Golang program memory leak?

Question

My golang program (url monitor) has a memory leak, it finially killed by kernel (oom). the env:

$ go version
go version go1.0.3

$ go env
GOARCH="amd64"
GOBIN=""
GOCHAR="6"
GOEXE=""
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread"
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/data/apps/go"
GOROOT="/usr/local/go"
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
CGO_ENABLED="1"

code:

package main

import (
    "bytes"
    "database/sql"
    "flag"
    "fmt"
    _ "github.com/Go-SQL-Driver/MySQL"
    "ijinshan.com/cfg"
    "log"
    "net"
    "net/http"
    "net/smtp"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

var (
    Log           *log.Logger
    Conf          cfg.KVConfig
    Debug         bool
    CpuCore       int
    HttpTransport = &http.Transport{
        Dial: func(netw, addr string) (net.Conn, error) {
            deadline := time.Now().Add(30 * time.Second)
            c, err := net.DialTimeout(netw, addr, 20*time.Second)
            if err != nil {
                return nil, err
            }

            c.SetDeadline(deadline)
            return c, nil
        },
        DisableKeepAlives: true,
    }
    HttpClient = &http.Client{
        Transport: HttpTransport,
    }
    WG            sync.WaitGroup
)

const (
    LogFileFlag   = os.O_WRONLY | os.O_CREATE | os.O_APPEND
    LogFileMode   = 0644
    LogFlag       = log.LstdFlags | log.Lshortfile
    GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum, 
    IFNULL(A.Website, ''), IFNULL(A.Descr, ''), 
    IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''), 
    IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '')
    FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200`

    HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名
    </td><td>tsid</td><td>章节</td><td>章节号</td><td>描述
    </td><td>videoid</td><td>网站</td><td>地址</td></tr>`
    HtmlTail = "</table>"
)

type videoInfo struct {
    name          string
    tsid          uint
    chapter       string
    chapterNum    uint
    descr         string
    videoId       string
    website       string
    androidWebUrl string
    iosWebUrl     string
    androidUrl    string
    androidUrl2   string
    iosUrl        string
}

func init() {
    var (
        confFile string
        err      error
    )

    // parse command argument:w
    flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path")
    flag.Parse()
    // read config
    if Conf, err = cfg.Read(confFile); err != nil {
        panic(fmt.Sprintf("Read config file \"%s\" failed (%s)",
            confFile, err.Error()))
    }
    // open log file
    file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode)
    if err != nil {
        panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"],
            err.Error()))
    }
    // init LOG
    Log = log.New(file, "", LogFlag)
    Debug = false
    i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32)
    if err != nil {
        panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"],
            err.Error()))
    }

    CpuCore = int(i)
}

func getHttpStatusCode(url string) int {
    if url == "" {
        return 200
    }

    req, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return 0
    }

    req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
    req.Header.Add("Connection", "close")
    resp, err := HttpClient.Do(req)
    if err != nil {
        return 0
    }

    defer resp.Body.Close()
    return resp.StatusCode
}

func sendMail(host, user, pwd, from, to, subject, body, mailType string) error {
    auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0])
    cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType)
    msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s",
        to, from, user, subject, cntType, body)

    return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg))
}

func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error {
    defer HttpTransport.CloseIdleConnections()
    db, err := sql.Open("mysql", Conf["weikan.mysql"])
    if err != nil {
        return err
    }

    rows, err := db.Query(GET_VIDEO_SQL)
    if err != nil {
        db.Close()
        return err
    }

    for rows.Next() {
        video := &videoInfo{}
        err = rows.Scan(&video.name, &video.tsid, &video.chapter,
            &video.chapterNum,
            &video.website, &video.descr, &video.videoId, &video.androidWebUrl,
            &video.iosWebUrl, &video.androidUrl, &video.androidUrl2,
            &video.iosUrl)
        if err != nil {
            db.Close()
            return err
        }

        videoChan <- video
        WG.Add(1)
    }

    db.Close()
    // wait check url finish
    WG.Wait()
    // send mail
    for {
        if htmlBuf.Len() == 0 {
            Log.Print("no error found!!!!!!!!")
            break
        }

        Log.Print("found error !!!!!!!!")
        /*
        err := sendMail("smtp.gmail.com:587", "xxxx",
            "xxx", "xxx <xxx>",
            Conf["mail.to"], "xxxxx",
            HtmlHead+htmlBuf.String()+HtmlTail, "html")
        if err != nil {
            Log.Printf("sendMail failed (%s)", err.Error())
            time.Sleep(10 * time.Second)
            continue
        }
        */

        Log.Print("send mail")
        break
    }

    Log.Print("reset buf")
    htmlBuf.Reset()
    return nil
}

func checkUrl(videoChan chan *videoInfo, errChan chan string) {
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    for {
        video := <-videoChan
        ok := true
        errUrl := ""

        if code := getHttpStatusCode(video.androidWebUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidWebUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.iosWebUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.iosWebUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.androidUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.androidUrl2); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidUrl2, code)
            ok = false
        }

        if code := getHttpStatusCode(video.iosUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.iosUrl, code)
            ok = false
        }

        if !ok {
            errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td>
            <td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
                video.name, video.tsid, video.chapter, video.chapterNum,
                video.descr, video.videoId,
                video.website, errUrl)
            Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name,
                video.chapter, video.descr)
        } else {
            Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name,
                video.chapter, video.descr)
            WG.Done()
        }
    }
}

func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) {
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    for {
        html := <-errChan
        _, err := htmlBuf.WriteString(html)
        if err != nil {
            Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html,
                err.Error())
            panic(err)
        }

        WG.Done()
    }
}

func main() {
    videoChan := make(chan *videoInfo, 100000)
    errChan := make(chan string, 100000)
    htmlBuf := &bytes.Buffer{}
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    // check url
    for i := 0; i < CpuCore; i++ {
        go checkUrl(videoChan, errChan)
    }
    // merge error string then send mail
    go mergeErr(errChan, htmlBuf)

    for {
        // get Video and LiveSrc video source
        if err := getVideos(videoChan, htmlBuf); err != nil {
            Log.Printf("getVideos failed (%s)", err.Error())
            time.Sleep(10 * time.Second)
            continue
        }

        // time.Sleep(1 * time.Hour)
    }

    Log.Print("exit...")
}

the code has four funcs:

getHttpStatusCode

free resource use resp.Body.Close()

sendMail

I don't need to free the resource manually

mergeErr

concat the err string by using a htmlBuf(*bytes.Buffer)

getVideos

First it gets the Video urls and then sends them to videoChan then it waits all the routines finish their check jobs. then sendmail and reset htmlBuf.

I don't find any resource that needs free, but.

$ top

shows:

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                                                                      
6451 root      20   0 3946m 115m 2808 S  0.7  0.2   6:11.20 vsmonitor

the VIRT and RES will grow ...

memory profiling:

(pprof) top
Total: 10.8 MB
2.3  21.2%  21.2%      2.3  21.2% main.main
2.0  18.5%  39.8%      2.0  18.5% bufio.NewWriterSize
1.5  13.9%  53.7%      1.5  13.9% bufio.NewReaderSize
1.5  13.9%  67.6%      1.5  13.9% compress/flate.NewReader
0.5   4.6%  72.2%      0.5   4.6% net.newFD
0.5   4.6%  76.8%      0.5   4.6% net.sockaddrToTCP
0.5   4.6%  81.5%      4.5  41.7% net/http.(*Transport).getConn
0.5   4.6%  86.1%      2.5  23.2% net/http.(*persistConn).readLoop
0.5   4.6%  90.7%      0.5   4.6% net/textproto.(*Reader).ReadMIMEHeader
0.5   4.6%  95.4%      0.5   4.6% net/url.(*URL).ResolveReference

Answer 1

It's pretty easy to add an option to your program so it'll record where the memory is being used. Nothing stood out to me in your program as terribly wrong. Are the files you download very large? Could you do a HEAD request instead? I've no idea if that'd help; if you have a high volume of requests maybe it would.

There is an (old-ish) article on the Go blog about the memory profiling at http://blog.golang.org/2011/06/profiling-go-programs.html and documentation at http://golang.org/pkg/runtime/pprof/ and http://golang.org/pkg/net/http/pprof/