Move Linux client & common packages into a public repo.

logtail/example/logadopt/logadopt.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+)
+
+func main() {
+	collection := flag.String("c", "", "logtail collection name")
+	publicID := flag.String("m", "", "machine public identifier")
+	apiKey := flag.String("p", "", "logtail API key")
+	flag.Parse()
+	if len(flag.Args()) != 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+	log.SetFlags(0)
+
+	req, err := http.NewRequest("POST", "https://log.tailscale.io/instances", strings.NewReader(url.Values{
+		"collection": []string{*collection},
+		"instances":  []string{*publicID},
+		"adopt":      []string{"true"},
+	}.Encode()))
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.SetBasicAuth(*apiKey, "")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		log.Fatalf("logadopt: response read failed %d: %v", resp.StatusCode, err)
+	}
+	if resp.StatusCode != 200 {
+		log.Fatalf("adoption failed: %d: %s", resp.StatusCode, string(b))
+	}
+	log.Printf("%s", string(b))
+}

logtail/example/logreprocess/demo.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+#
+# This shell script demonstrates writing logs from machines
+# and then reprocessing those logs to amalgamate python tracebacks
+# into a single log entry in a new collection.
+#
+# To run this demo, first install the example applications:
+#
+#        go install tailscale.com/logtail/example/...
+#
+# Then generate a LOGTAIL_API_KEY and two test collections by visiting:
+#
+#        https://log.tailscale.io
+#
+# Then set the three variables below.
+trap 'rv=$?; [ "$rv" = 0 ] || echo "-- exiting with code $rv"; exit $rv' EXIT
+set -e
+
+LOG_TEXT='server starting
+config file loaded
+answering queries
+Traceback (most recent call last):
+  File "/Users/crawshaw/junk.py", line 6, in <module>
+    main()
+  File "/Users/crawshaw/junk.py", line 4, in main
+    raise Exception("oops")
+Exception: oops'
+
+die() {
+	echo "$0: $*" >&2
+	exit 1
+}
+
+msg() {
+	echo "-- $*" >&2
+}
+
+if [ -z "$LOGTAIL_API_KEY" ]; then
+	die "LOGTAIL_API_KEY is not set"
+fi
+
+if [ -z "$COLLECTION_IN" ]; then
+	die "COLLECTION_IN is not set"
+fi
+
+if [ -z "$COLLECTION_OUT" ]; then
+	die "COLLECTION_OUT is not set"
+fi
+
+# Private IDs are 32-bytes of random hex.
+# Normally you'd keep the same private IDs from one run to the next, but
+# this is just an example.
+msg "Generating keys..."
+privateid1=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+privateid2=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+privateid3=$(hexdump -n 32 -e '8/4 "%08X"' /dev/urandom)
+
+# Public IDs are the SHA-256 of the private ID.
+publicid1=$(echo -n $privateid1 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+publicid2=$(echo -n $privateid2 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+publicid3=$(echo -n $privateid3 | xxd -r -p - | shasum -a 256 | sed 's/ -//')
+
+# Write the machine logs to the input collection.
+# Notice that this doesn't require an API key.
+msg "Producing new logs..."
+echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid1 >/dev/null
+echo "$LOG_TEXT" | logtail -c $COLLECTION_IN -k $privateid2 >/dev/null
+
+# Adopt the logs, so they will be kept and are readable.
+msg "Adopting logs..."
+logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid1
+logadopt -p "$LOGTAIL_API_KEY" -c "$COLLECTION_IN" -m $publicid2
+
+# Reprocess the logs, amalgamating python tracebacks.
+#
+# We'll take that reprocessed output and write it to a separate collection,
+# again via logtail.
+#
+# Time out quickly because all our "interesting" logs (generated
+# above) have already been processed.
+msg "Reprocessing logs..."
+logreprocess -t 3s -c "$COLLECTION_IN" -p "$LOGTAIL_API_KEY" 2>&1 |
+  logtail -c "$COLLECTION_OUT" -k $privateid3

logtail/example/logreprocess/logreprocess.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The logreprocess program tails a log and reprocesses it.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"tailscale.com/logtail"
+)
+
+func main() {
+	collection := flag.String("c", "", "logtail collection name to read")
+	apiKey := flag.String("p", "", "logtail API key")
+	timeout := flag.Duration("t", 0, "timeout after which logreprocess quits")
+	flag.Parse()
+	if len(flag.Args()) != 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+	log.SetFlags(0)
+
+	if *timeout != 0 {
+		go func() {
+			<-time.After(*timeout)
+			log.Printf("logreprocess: timeout reached, quitting")
+			os.Exit(1)
+		}()
+	}
+
+	req, err := http.NewRequest("GET", "https://log.tailscale.io/c/"+*collection+"?stream=true", nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.SetBasicAuth(*apiKey, "")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			log.Fatalf("logreprocess: read error %d: %v", resp.StatusCode, err)
+		}
+		log.Fatalf("logreprocess: read error %d: %s", resp.StatusCode, string(b))
+	}
+
+	tracebackCache := make(map[logtail.PublicID]*ProcessedMsg)
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		var msg Msg
+		if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
+			log.Fatalf("logreprocess of %q: %v", string(scanner.Bytes()), err)
+		}
+		var pMsg *ProcessedMsg
+		if pMsg = tracebackCache[msg.Logtail.Instance]; pMsg != nil {
+			pMsg.Text += "\n" + msg.Text
+			if strings.HasPrefix(msg.Text, "Exception: ") {
+				delete(tracebackCache, msg.Logtail.Instance)
+			} else {
+				continue // write later
+			}
+		} else {
+			pMsg = &ProcessedMsg{
+				OrigInstance: msg.Logtail.Instance,
+				Text:         msg.Text,
+			}
+			pMsg.Logtail.ClientTime = msg.Logtail.ClientTime
+		}
+
+		if strings.HasPrefix(msg.Text, "Traceback (most recent call last):") {
+			tracebackCache[msg.Logtail.Instance] = pMsg
+			continue // write later
+		}
+
+		b, err := json.Marshal(pMsg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Printf("%s", b)
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+type Msg struct {
+	Logtail struct {
+		Instance   logtail.PublicID `json:"instance"`
+		ClientTime time.Time        `json:"client_time"`
+	} `json:"logtail"`
+
+	Text string `json:"text"`
+}
+
+type ProcessedMsg struct {
+	Logtail struct {
+		ClientTime time.Time `json:"client_time"`
+	} `json:"logtail"`
+
+	OrigInstance logtail.PublicID `json:"orig_instance"`
+	Text         string           `json:"text"`
+}

logtail/example/logtail/logtail.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The logtail program logs stdin.
+package main
+
+import (
+	"bufio"
+	"flag"
+	"io"
+	"log"
+	"os"
+
+	"tailscale.com/logtail"
+)
+
+func main() {
+	collection := flag.String("c", "", "logtail collection name")
+	privateID := flag.String("k", "", "machine private identifier, 32-bytes in hex")
+	flag.Parse()
+	if len(flag.Args()) != 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	log.SetFlags(0)
+
+	var id logtail.PrivateID
+	if err := id.UnmarshalText([]byte(*privateID)); err != nil {
+		log.Fatalf("logtail: bad -privateid: %v", err)
+	}
+
+	logger := logtail.Log(logtail.Config{
+		Collection: *collection,
+		PrivateID:  id,
+	})
+	log.SetOutput(io.MultiWriter(logger, os.Stdout))
+	defer logger.Flush()
+	defer log.Printf("logtail exited")
+
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		log.Println(scanner.Text())
+	}
+}