Initial commit
This commit is contained in:
commit
9f280a7883
3 changed files with 316 additions and 0 deletions
85
README.md
Normal file
85
README.md
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# Purpose
|
||||
|
||||
`Vlog` extends the structured log JSONHandler with sending logs to a VictoriaLogs instance for central logging.
|
||||
|
||||
Logs are first queued to a buffered channel processed by go-routine loop which transform and send the log to the server.
|
||||
|
||||
In case of network error, queue getting full - anything that doesn't get a HTTP 200 from VictoriaLogs; logs get stored to disk to be processed at intervals.
|
||||
|
||||
Streams are set to separate application runs to instances of it running on different servers.
|
||||
|
||||
# Usage
|
||||
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
// External
|
||||
"git.gibonuddevalla.se/go/vlog"
|
||||
|
||||
// Standard
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
opts := slog.HandlerOptions{
|
||||
Level: slog.LevelInfo, // Minimum level to display.
|
||||
}
|
||||
|
||||
handler := vlog.New(
|
||||
os.Stdout,
|
||||
opts,
|
||||
"/tmp/vlog_spool", // Spool directory when logs can't be sent.
|
||||
"https://log.hum.se/", // URL to VictoriaLogs instance.
|
||||
"vlog", // Application name.
|
||||
"srv01", // System/server that runs the instance.
|
||||
"dev", // Instance name.
|
||||
)
|
||||
|
||||
log := slog.New(handler)
|
||||
|
||||
for {
|
||||
log.Warn("server", "foo", "bar", "baz", 123)
|
||||
time.Sleep(time.Second * 1)
|
||||
}
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
# Logging stream
|
||||
|
||||
The stream is created with:
|
||||
1) application name.
|
||||
1) system name running the software.
|
||||
1) instance name (like dev or prod).
|
||||
1) run, which is automated to YYMMDD_HHMMSS when application is started.
|
||||
|
||||
These are prefixed with `log`, see JSON data below.
|
||||
|
||||
# Logging data
|
||||
|
||||
Application log attributes (`foo`, `baz` in the example code) are added to the top level log entry.
|
||||
|
||||
The vlog library adds go build information (git repo and such) and a runtime trace when sending logs with level `warn`, `warn` or `error`.
|
||||
|
||||
The `info` level is excepted to this since it would probably be used the most for regular data that wouldn't lead to any troubleshooting. Making the trace eats a bit of performance.
|
||||
|
||||
```json
|
||||
{
|
||||
"_time": "2026-02-14T08:43:02.760194943Z",
|
||||
"_stream_id": "0000000000000000926515688d74d64c8facef73b36a02b5",
|
||||
"_stream": "{log.application=\"vlog\",log.instance=\"dev\",log.run=\"260214_094242\",log.system=\"srv01\"}",
|
||||
"_msg": "server",
|
||||
"baz": "123",
|
||||
"foo": "bar",
|
||||
"log.application": "vlog",
|
||||
"log.instance": "dev",
|
||||
"log.level": "WARN",
|
||||
"log.run": "260214_094242",
|
||||
"log.system": "srv01",
|
||||
"log.build": "{\"GoVersion\":\"go1.25.4\",\"Path\":\"hum\",\"Main\":{\"Path\":\"hum\",\"Version\":\"(devel)\"},\"Deps\":[{\"Path\":\"git.gibonuddevalla.se/go/vlog\",\"Version\":\"(devel)\"}],\"Settings\":[{\"Key\":\"-buildmode\",\"Value\":\"exe\"},{\"Key\":\"-compiler\",\"Value\":\"gc\"},{\"Key\":\"CGO_ENABLED\",\"Value\":\"1\"},{\"Key\":\"CGO_CFLAGS\"},{\"Key\":\"CGO_CPPFLAGS\"},{\"Key\":\"CGO_CXXFLAGS\"},{\"Key\":\"CGO_LDFLAGS\"},{\"Key\":\"GOARCH\",\"Value\":\"amd64\"},{\"Key\":\"GOOS\",\"Value\":\"linux\"},{\"Key\":\"GOAMD64\",\"Value\":\"v1\"}]}",
|
||||
"log.trace": "goroutine 1 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:26 +0x5e\ngit.gibonuddevalla.se/go/vlog.(*VictoriaHandler).queue(_, {{0xc25c2b45ad4fa77f, 0x4a9205feb, 0x9bf140}, {0x7339a5, 0x6}, 0x4, 0x65125f, {{{0x73345d, 0x3}, ...}, ...}, ...})\n\t/tmp/foo/vlog/lib.go:146 +0x65\ngit.gibonuddevalla.se/go/vlog.VictoriaHandler.Handle({0xc000074080, {0x7335b3, 0x4}, {0x733793, 0x5}, {0x733466, 0x3}, {0x73842d, 0x14}, {0x7aec30, ...}, ...}, ...)\n\t/tmp/foo/vlog/lib.go:75 +0x65\nlog/slog.(*Logger).log(0xc0000f3f30, {0x7aebf8?, 0x9e03c0?}, 0x4, {0x7339a5, 0x6}, {0xc0000f3ef0, 0x4, 0x4})\n\t/usr/local/go/src/log/slog/logger.go:256 +0x208\nlog/slog.(*Logger).Warn(...)\n\t/usr/local/go/src/log/slog/logger.go:219\nmain.main()\n\t/tmp/foo/hum/main.go:31 +0x1a6\n"
|
||||
}
|
||||
```
|
||||
3
go.mod
Normal file
3
go.mod
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module git.gibonuddevalla.se/go/vlog
|
||||
|
||||
go 1.25.4
|
||||
228
lib.go
Normal file
228
lib.go
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
package vlog
|
||||
|
||||
import (
|
||||
// Standard
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VictoriaHandler struct {
|
||||
*slog.JSONHandler
|
||||
Application string
|
||||
System string
|
||||
Instance string
|
||||
URL string
|
||||
|
||||
handler slog.Handler
|
||||
filecounter int
|
||||
logdir string
|
||||
logQueue chan []byte
|
||||
run string
|
||||
log *slog.Logger
|
||||
httpClient http.Client
|
||||
}
|
||||
|
||||
type LogData struct {
|
||||
Level string
|
||||
Msg string
|
||||
Args []any
|
||||
}
|
||||
|
||||
func New(w io.Writer, opts slog.HandlerOptions, logdir, url, application, system, instance string) (vl VictoriaHandler) { // {{{
|
||||
var err error
|
||||
vl.JSONHandler = slog.NewJSONHandler(w, &opts)
|
||||
|
||||
vl.httpClient = http.Client{}
|
||||
vl.logQueue = make(chan []byte, 8192)
|
||||
vl.logdir = logdir
|
||||
|
||||
vl.handler = slog.NewJSONHandler(w, &opts)
|
||||
|
||||
// logdir is used to spool logfiles in case of
|
||||
err = os.MkdirAll(logdir, 0700)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
vl.log = slog.New(slog.NewJSONHandler(w, &opts))
|
||||
vl.log.Debug("VICTORIALOG", "tempory_storage", vl.logdir)
|
||||
|
||||
vl.URL = url
|
||||
vl.Application = application
|
||||
vl.System = system
|
||||
vl.Instance = instance
|
||||
vl.run = time.Now().Format("060102_150405")
|
||||
|
||||
go vl.queueHandler()
|
||||
go vl.diskHandler()
|
||||
return
|
||||
} // }}}
|
||||
|
||||
// Handle is overridden to enable central logging.
|
||||
func (vl VictoriaHandler) Handle(ctx context.Context, record slog.Record) error {// {{{
|
||||
vl.queue(record)
|
||||
return vl.handler.Handle(ctx, record)
|
||||
}// }}}
|
||||
|
||||
func (vl *VictoriaHandler) diskHandler() { // {{{
|
||||
for {
|
||||
time.Sleep(time.Second * 5)
|
||||
|
||||
err := filepath.WalkDir(vl.logdir, func(fname string, d fs.DirEntry, err error) error {
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(fname)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
err = vl.send(data)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
err = os.Remove(fname)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
}
|
||||
}
|
||||
} // }}}
|
||||
func (vl *VictoriaHandler) queueHandler() { // {{{
|
||||
var err error
|
||||
for {
|
||||
msg := <-vl.logQueue
|
||||
err = vl.send(msg)
|
||||
if err != nil {
|
||||
vl.logToFile(msg)
|
||||
}
|
||||
}
|
||||
} // }}}
|
||||
|
||||
func (vl *VictoriaHandler) queue(rec slog.Record) { // {{{
|
||||
/*
|
||||
{
|
||||
"_msg": "hello world"
|
||||
"_time": "2026-02-12T09:39:45.374813969Z",
|
||||
"log": {
|
||||
"level": "info",
|
||||
"application": "transcode",
|
||||
"system": "server01.hum.ding",
|
||||
"instance": "production",
|
||||
"run": "260212_123000",
|
||||
build: "",
|
||||
trace: "",
|
||||
},
|
||||
"<app_attr>": <data>
|
||||
}
|
||||
*/
|
||||
data := make(map[string]any)
|
||||
dataLog := make(map[string]string)
|
||||
|
||||
if rec.Level == slog.LevelDebug || rec.Level == slog.LevelWarn || rec.Level == slog.LevelError {
|
||||
trace := debug.Stack()
|
||||
buildInfo, _ := debug.ReadBuildInfo()
|
||||
buildBytes, _ := json.Marshal(buildInfo)
|
||||
dataLog["trace"] = string(trace)
|
||||
dataLog["build"] = string(buildBytes)
|
||||
}
|
||||
|
||||
dataLog["level"] = rec.Level.String()
|
||||
dataLog["application"] = vl.Application
|
||||
dataLog["system"] = vl.System
|
||||
dataLog["instance"] = vl.Instance
|
||||
dataLog["run"] = vl.run
|
||||
|
||||
data["_msg"] = rec.Message
|
||||
data["_time"] = rec.Time.Format(time.RFC3339Nano)
|
||||
data["log"] = dataLog
|
||||
|
||||
rec.Attrs(func(attr slog.Attr) bool {
|
||||
data[attr.Key] = attr.Value.Any()
|
||||
return true
|
||||
})
|
||||
|
||||
// Data is marshalled into its final form to be queued and sent.
|
||||
j, _ := json.Marshal(data)
|
||||
|
||||
select {
|
||||
case vl.logQueue <- j:
|
||||
vl.log.Debug("VICTORIALOG", "op", "sent to channel")
|
||||
default:
|
||||
vl.logToFile(j)
|
||||
vl.log.Debug("VICTORIALOG", "op", "sent to file")
|
||||
}
|
||||
} // }}}
|
||||
func (vl *VictoriaHandler) send(data []byte) (err error) { // {{{
|
||||
request := bytes.NewReader(data)
|
||||
|
||||
// URL is composed with path query cleanly built to
|
||||
// ensure correct encoding of it.
|
||||
var vlogURL *url.URL
|
||||
vlogURL, err = url.Parse(vl.URL + "/insert/jsonline")
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return
|
||||
}
|
||||
values := vlogURL.Query()
|
||||
values.Add("_time_field", "_time")
|
||||
values.Add("_stream_fields", "log.application,log.system,log.instance,log.run")
|
||||
vlogURL.RawQuery = values.Encode()
|
||||
|
||||
// Request is sent to server in order to be centrally lagged.
|
||||
var req *http.Request
|
||||
req, err = http.NewRequest("POST", vlogURL.String(), request)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
resp, err = vl.httpClient.Do(req)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
vl.log.Error("VICTORIALOG", "error", resp.Status, "body", string(body))
|
||||
return fmt.Errorf("Unsuccesful status code, %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
vl.log.Debug("VICTORIALOG", "op", "sent to server")
|
||||
return
|
||||
} // }}}
|
||||
func (vl *VictoriaHandler) logToFile(data []byte) { // {{{
|
||||
vl.filecounter++
|
||||
fname := path.Join(vl.logdir, fmt.Sprintf("%010X.log", vl.filecounter))
|
||||
err := os.WriteFile(fname, data, 0600)
|
||||
if err != nil {
|
||||
vl.log.Error("VICTORIALOG", "error", err)
|
||||
return
|
||||
}
|
||||
vl.log.Error("VICTORIALOG", "op", "sent to disk", "fname", fname)
|
||||
} // }}}
|
||||
Loading…
Add table
Reference in a new issue