From 9f280a788353b320fbd89ec3fa67815fd07c5230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20=C3=85hall?= Date: Sat, 14 Feb 2026 10:02:32 +0100 Subject: [PATCH] Initial commit --- README.md | 85 ++++++++++++++++++++ go.mod | 3 + lib.go | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+) create mode 100644 README.md create mode 100644 go.mod create mode 100644 lib.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..1408b1a --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +# Purpose + +`Vlog` extends the structured log JSONHandler with sending logs to a VictoriaLogs instance for central logging. + +Logs are first queued to a buffered channel processed by go-routine loop which transform and send the log to the server. + +In case of network error, queue getting full - anything that doesn't get a HTTP 200 from VictoriaLogs; logs get stored to disk to be processed at intervals. + +Streams are set to separate application runs to instances of it running on different servers. + +# Usage + +```golang +package main + +import ( + // External + "git.gibonuddevalla.se/go/vlog" + + // Standard + "log/slog" + "os" + "time" +) + +func main() { + opts := slog.HandlerOptions{ + Level: slog.LevelInfo, // Minimum level to display. + } + + handler := vlog.New( + os.Stdout, + opts, + "/tmp/vlog_spool", // Spool directory when logs can't be sent. + "https://log.hum.se/", // URL to VictoriaLogs instance. + "vlog", // Application name. + "srv01", // System/server that runs the instance. + "dev", // Instance name. + ) + + log := slog.New(handler) + + for { + log.Warn("server", "foo", "bar", "baz", 123) + time.Sleep(time.Second * 1) + } + +} +``` + +# Logging stream + +The stream is created with: +1) application name. +1) system name running the software. +1) instance name (like dev or prod). +1) run, which is automated to YYMMDD_HHMMSS when application is started. + +These are prefixed with `log`, see JSON data below. + +# Logging data + +Application log attributes (`foo`, `baz` in the example code) are added to the top level log entry. + +The vlog library adds go build information (git repo and such) and a runtime trace when sending logs with level `warn`, `warn` or `error`. + +The `info` level is excepted to this since it would probably be used the most for regular data that wouldn't lead to any troubleshooting. Making the trace eats a bit of performance. + +```json +{ + "_time": "2026-02-14T08:43:02.760194943Z", + "_stream_id": "0000000000000000926515688d74d64c8facef73b36a02b5", + "_stream": "{log.application=\"vlog\",log.instance=\"dev\",log.run=\"260214_094242\",log.system=\"srv01\"}", + "_msg": "server", + "baz": "123", + "foo": "bar", + "log.application": "vlog", + "log.instance": "dev", + "log.level": "WARN", + "log.run": "260214_094242", + "log.system": "srv01", + "log.build": "{\"GoVersion\":\"go1.25.4\",\"Path\":\"hum\",\"Main\":{\"Path\":\"hum\",\"Version\":\"(devel)\"},\"Deps\":[{\"Path\":\"git.gibonuddevalla.se/go/vlog\",\"Version\":\"(devel)\"}],\"Settings\":[{\"Key\":\"-buildmode\",\"Value\":\"exe\"},{\"Key\":\"-compiler\",\"Value\":\"gc\"},{\"Key\":\"CGO_ENABLED\",\"Value\":\"1\"},{\"Key\":\"CGO_CFLAGS\"},{\"Key\":\"CGO_CPPFLAGS\"},{\"Key\":\"CGO_CXXFLAGS\"},{\"Key\":\"CGO_LDFLAGS\"},{\"Key\":\"GOARCH\",\"Value\":\"amd64\"},{\"Key\":\"GOOS\",\"Value\":\"linux\"},{\"Key\":\"GOAMD64\",\"Value\":\"v1\"}]}", + "log.trace": "goroutine 1 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:26 +0x5e\ngit.gibonuddevalla.se/go/vlog.(*VictoriaHandler).queue(_, {{0xc25c2b45ad4fa77f, 0x4a9205feb, 0x9bf140}, {0x7339a5, 0x6}, 0x4, 0x65125f, {{{0x73345d, 0x3}, ...}, ...}, ...})\n\t/tmp/foo/vlog/lib.go:146 +0x65\ngit.gibonuddevalla.se/go/vlog.VictoriaHandler.Handle({0xc000074080, {0x7335b3, 0x4}, {0x733793, 0x5}, {0x733466, 0x3}, {0x73842d, 0x14}, {0x7aec30, ...}, ...}, ...)\n\t/tmp/foo/vlog/lib.go:75 +0x65\nlog/slog.(*Logger).log(0xc0000f3f30, {0x7aebf8?, 0x9e03c0?}, 0x4, {0x7339a5, 0x6}, {0xc0000f3ef0, 0x4, 0x4})\n\t/usr/local/go/src/log/slog/logger.go:256 +0x208\nlog/slog.(*Logger).Warn(...)\n\t/usr/local/go/src/log/slog/logger.go:219\nmain.main()\n\t/tmp/foo/hum/main.go:31 +0x1a6\n" +} +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6d2666e --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.gibonuddevalla.se/go/vlog + +go 1.25.4 diff --git a/lib.go b/lib.go new file mode 100644 index 0000000..5139676 --- /dev/null +++ b/lib.go @@ -0,0 +1,228 @@ +package vlog + +import ( + // Standard + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "runtime/debug" + "time" +) + +type VictoriaHandler struct { + *slog.JSONHandler + Application string + System string + Instance string + URL string + + handler slog.Handler + filecounter int + logdir string + logQueue chan []byte + run string + log *slog.Logger + httpClient http.Client +} + +type LogData struct { + Level string + Msg string + Args []any +} + +func New(w io.Writer, opts slog.HandlerOptions, logdir, url, application, system, instance string) (vl VictoriaHandler) { // {{{ + var err error + vl.JSONHandler = slog.NewJSONHandler(w, &opts) + + vl.httpClient = http.Client{} + vl.logQueue = make(chan []byte, 8192) + vl.logdir = logdir + + vl.handler = slog.NewJSONHandler(w, &opts) + + // logdir is used to spool logfiles in case of + err = os.MkdirAll(logdir, 0700) + if err != nil { + panic(err) + } + + vl.log = slog.New(slog.NewJSONHandler(w, &opts)) + vl.log.Debug("VICTORIALOG", "tempory_storage", vl.logdir) + + vl.URL = url + vl.Application = application + vl.System = system + vl.Instance = instance + vl.run = time.Now().Format("060102_150405") + + go vl.queueHandler() + go vl.diskHandler() + return +} // }}} + +// Handle is overridden to enable central logging. +func (vl VictoriaHandler) Handle(ctx context.Context, record slog.Record) error {// {{{ + vl.queue(record) + return vl.handler.Handle(ctx, record) +}// }}} + +func (vl *VictoriaHandler) diskHandler() { // {{{ + for { + time.Sleep(time.Second * 5) + + err := filepath.WalkDir(vl.logdir, func(fname string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + + data, err := os.ReadFile(fname) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return nil + } + + err = vl.send(data) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return nil + } + + err = os.Remove(fname) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return nil + } + + return nil + }) + + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + } + } +} // }}} +func (vl *VictoriaHandler) queueHandler() { // {{{ + var err error + for { + msg := <-vl.logQueue + err = vl.send(msg) + if err != nil { + vl.logToFile(msg) + } + } +} // }}} + +func (vl *VictoriaHandler) queue(rec slog.Record) { // {{{ + /* + { + "_msg": "hello world" + "_time": "2026-02-12T09:39:45.374813969Z", + "log": { + "level": "info", + "application": "transcode", + "system": "server01.hum.ding", + "instance": "production", + "run": "260212_123000", + build: "", + trace: "", + }, + "": + } + */ + data := make(map[string]any) + dataLog := make(map[string]string) + + if rec.Level == slog.LevelDebug || rec.Level == slog.LevelWarn || rec.Level == slog.LevelError { + trace := debug.Stack() + buildInfo, _ := debug.ReadBuildInfo() + buildBytes, _ := json.Marshal(buildInfo) + dataLog["trace"] = string(trace) + dataLog["build"] = string(buildBytes) + } + + dataLog["level"] = rec.Level.String() + dataLog["application"] = vl.Application + dataLog["system"] = vl.System + dataLog["instance"] = vl.Instance + dataLog["run"] = vl.run + + data["_msg"] = rec.Message + data["_time"] = rec.Time.Format(time.RFC3339Nano) + data["log"] = dataLog + + rec.Attrs(func(attr slog.Attr) bool { + data[attr.Key] = attr.Value.Any() + return true + }) + + // Data is marshalled into its final form to be queued and sent. + j, _ := json.Marshal(data) + + select { + case vl.logQueue <- j: + vl.log.Debug("VICTORIALOG", "op", "sent to channel") + default: + vl.logToFile(j) + vl.log.Debug("VICTORIALOG", "op", "sent to file") + } +} // }}} +func (vl *VictoriaHandler) send(data []byte) (err error) { // {{{ + request := bytes.NewReader(data) + + // URL is composed with path query cleanly built to + // ensure correct encoding of it. + var vlogURL *url.URL + vlogURL, err = url.Parse(vl.URL + "/insert/jsonline") + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return + } + values := vlogURL.Query() + values.Add("_time_field", "_time") + values.Add("_stream_fields", "log.application,log.system,log.instance,log.run") + vlogURL.RawQuery = values.Encode() + + // Request is sent to server in order to be centrally lagged. + var req *http.Request + req, err = http.NewRequest("POST", vlogURL.String(), request) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return + } + + var resp *http.Response + resp, err = vl.httpClient.Do(req) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return + } + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(resp.Body) + vl.log.Error("VICTORIALOG", "error", resp.Status, "body", string(body)) + return fmt.Errorf("Unsuccesful status code, %d", resp.StatusCode) + } + + vl.log.Debug("VICTORIALOG", "op", "sent to server") + return +} // }}} +func (vl *VictoriaHandler) logToFile(data []byte) { // {{{ + vl.filecounter++ + fname := path.Join(vl.logdir, fmt.Sprintf("%010X.log", vl.filecounter)) + err := os.WriteFile(fname, data, 0600) + if err != nil { + vl.log.Error("VICTORIALOG", "error", err) + return + } + vl.log.Error("VICTORIALOG", "op", "sent to disk", "fname", fname) +} // }}}