Vendor dependencies for GCS

vendor/cloud.google.com/go/bigtable/internal/cbtconfig/cbtconfig.go

@@ -0,0 +1,246 @@
+/*
+Copyright 2015 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cbtconfig encapsulates common code for reading configuration from .cbtrc and gcloud.
+package cbtconfig
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"golang.org/x/oauth2"
+	"google.golang.org/grpc/credentials"
+)
+
+// Config represents a configuration.
+type Config struct {
+	Project, Instance string                           // required
+	Creds             string                           // optional
+	AdminEndpoint     string                           // optional
+	DataEndpoint      string                           // optional
+	CertFile          string                           // optional
+	TokenSource       oauth2.TokenSource               // derived
+	TLSCreds          credentials.TransportCredentials // derived
+}
+
+type RequiredFlags uint
+
+const NoneRequired RequiredFlags = 0
+const (
+	ProjectRequired RequiredFlags = 1 << iota
+	InstanceRequired
+)
+const ProjectAndInstanceRequired RequiredFlags = ProjectRequired | InstanceRequired
+
+// RegisterFlags registers a set of standard flags for this config.
+// It should be called before flag.Parse.
+func (c *Config) RegisterFlags() {
+	flag.StringVar(&c.Project, "project", c.Project, "project ID, if unset uses gcloud configured project")
+	flag.StringVar(&c.Instance, "instance", c.Instance, "Cloud Bigtable instance")
+	flag.StringVar(&c.Creds, "creds", c.Creds, "if set, use application credentials in this file")
+	flag.StringVar(&c.AdminEndpoint, "admin-endpoint", c.AdminEndpoint, "Override the admin api endpoint")
+	flag.StringVar(&c.DataEndpoint, "data-endpoint", c.DataEndpoint, "Override the data api endpoint")
+	flag.StringVar(&c.CertFile, "cert-file", c.CertFile, "Override the TLS certificates file")
+}
+
+// CheckFlags checks that the required config values are set.
+func (c *Config) CheckFlags(required RequiredFlags) error {
+	var missing []string
+	if c.CertFile != "" {
+		b, err := ioutil.ReadFile(c.CertFile)
+		if err != nil {
+			return fmt.Errorf("Failed to load certificates from %s: %v", c.CertFile, err)
+		}
+
+		cp := x509.NewCertPool()
+		if !cp.AppendCertsFromPEM(b) {
+			return fmt.Errorf("Failed to append certificates from %s", c.CertFile)
+		}
+
+		c.TLSCreds = credentials.NewTLS(&tls.Config{RootCAs: cp})
+	}
+	if required != NoneRequired {
+		c.SetFromGcloud()
+	}
+	if required&ProjectRequired != 0 && c.Project == "" {
+		missing = append(missing, "-project")
+	}
+	if required&InstanceRequired != 0 && c.Instance == "" {
+		missing = append(missing, "-instance")
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("Missing %s", strings.Join(missing, " and "))
+	}
+	return nil
+}
+
+// Filename returns the filename consulted for standard configuration.
+func Filename() string {
+	// TODO(dsymonds): Might need tweaking for Windows.
+	return filepath.Join(os.Getenv("HOME"), ".cbtrc")
+}
+
+// Load loads a .cbtrc file.
+// If the file is not present, an empty config is returned.
+func Load() (*Config, error) {
+	filename := Filename()
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		// silent fail if the file isn't there
+		if os.IsNotExist(err) {
+			return &Config{}, nil
+		}
+		return nil, fmt.Errorf("Reading %s: %v", filename, err)
+	}
+	c := new(Config)
+	s := bufio.NewScanner(bytes.NewReader(data))
+	for s.Scan() {
+		line := s.Text()
+		i := strings.Index(line, "=")
+		if i < 0 {
+			return nil, fmt.Errorf("Bad line in %s: %q", filename, line)
+		}
+		key, val := strings.TrimSpace(line[:i]), strings.TrimSpace(line[i+1:])
+		switch key {
+		default:
+			return nil, fmt.Errorf("Unknown key in %s: %q", filename, key)
+		case "project":
+			c.Project = val
+		case "instance":
+			c.Instance = val
+		case "creds":
+			c.Creds = val
+		case "admin-endpoint":
+			c.AdminEndpoint = val
+		case "data-endpoint":
+			c.DataEndpoint = val
+		}
+
+	}
+	return c, s.Err()
+}
+
+type GcloudCredential struct {
+	AccessToken string    `json:"access_token"`
+	Expiry      time.Time `json:"token_expiry"`
+}
+
+func (cred *GcloudCredential) Token() *oauth2.Token {
+	return &oauth2.Token{AccessToken: cred.AccessToken, TokenType: "Bearer", Expiry: cred.Expiry}
+}
+
+type GcloudConfig struct {
+	Configuration struct {
+		Properties struct {
+			Core struct {
+				Project string `json:"project"`
+			} `json:"core"`
+		} `json:"properties"`
+	} `json:"configuration"`
+	Credential GcloudCredential `json:"credential"`
+}
+
+type GcloudCmdTokenSource struct {
+	Command string
+	Args    []string
+}
+
+// Token implements the oauth2.TokenSource interface
+func (g *GcloudCmdTokenSource) Token() (*oauth2.Token, error) {
+	gcloudConfig, err := LoadGcloudConfig(g.Command, g.Args)
+	if err != nil {
+		return nil, err
+	}
+	return gcloudConfig.Credential.Token(), nil
+}
+
+// LoadGcloudConfig retrieves the gcloud configuration values we need use via the
+// 'config-helper' command
+func LoadGcloudConfig(gcloudCmd string, gcloudCmdArgs []string) (*GcloudConfig, error) {
+	out, err := exec.Command(gcloudCmd, gcloudCmdArgs...).Output()
+	if err != nil {
+		return nil, fmt.Errorf("Could not retrieve gcloud configuration")
+	}
+
+	var gcloudConfig GcloudConfig
+	if err := json.Unmarshal(out, &gcloudConfig); err != nil {
+		return nil, fmt.Errorf("Could not parse gcloud configuration")
+	}
+
+	return &gcloudConfig, nil
+}
+
+// SetFromGcloud retrieves and sets any missing config values from the gcloud
+// configuration if possible possible
+func (c *Config) SetFromGcloud() error {
+
+	if c.Creds == "" {
+		c.Creds = os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+		if c.Creds == "" {
+			log.Printf("-creds flag unset, will use gcloud credential")
+		}
+	} else {
+		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", c.Creds)
+	}
+
+	if c.Project == "" {
+		log.Printf("-project flag unset, will use gcloud active project")
+	}
+
+	if c.Creds != "" && c.Project != "" {
+		return nil
+	}
+
+	gcloudCmd := "gcloud"
+	if runtime.GOOS == "windows" {
+		gcloudCmd = gcloudCmd + ".cmd"
+	}
+
+	gcloudCmdArgs := []string{"config", "config-helper",
+		"--format=json(configuration.properties.core.project,credential)"}
+
+	gcloudConfig, err := LoadGcloudConfig(gcloudCmd, gcloudCmdArgs)
+	if err != nil {
+		return err
+	}
+
+	if c.Project == "" && gcloudConfig.Configuration.Properties.Core.Project != "" {
+		log.Printf("gcloud active project is \"%s\"",
+			gcloudConfig.Configuration.Properties.Core.Project)
+		c.Project = gcloudConfig.Configuration.Properties.Core.Project
+	}
+
+	if c.Creds == "" {
+		c.TokenSource = oauth2.ReuseTokenSource(
+			gcloudConfig.Credential.Token(),
+			&GcloudCmdTokenSource{Command: gcloudCmd, Args: gcloudCmdArgs})
+	}
+
+	return nil
+}

vendor/cloud.google.com/go/bigtable/internal/gax/call_option.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This is ia snapshot from github.com/googleapis/gax-go with minor modifications.
+package gax
+
+import (
+	"time"
+
+	"google.golang.org/grpc/codes"
+)
+
+type CallOption interface {
+	Resolve(*CallSettings)
+}
+
+type callOptions []CallOption
+
+func (opts callOptions) Resolve(s *CallSettings) *CallSettings {
+	for _, opt := range opts {
+		opt.Resolve(s)
+	}
+	return s
+}
+
+// Encapsulates the call settings for a particular API call.
+type CallSettings struct {
+	Timeout       time.Duration
+	RetrySettings RetrySettings
+}
+
+// Per-call configurable settings for retrying upon transient failure.
+type RetrySettings struct {
+	RetryCodes      map[codes.Code]bool
+	BackoffSettings BackoffSettings
+}
+
+// Parameters to the exponential backoff algorithm for retrying.
+type BackoffSettings struct {
+	DelayTimeoutSettings MultipliableDuration
+	RPCTimeoutSettings   MultipliableDuration
+}
+
+type MultipliableDuration struct {
+	Initial    time.Duration
+	Max        time.Duration
+	Multiplier float64
+}
+
+func (w CallSettings) Resolve(s *CallSettings) {
+	s.Timeout = w.Timeout
+	s.RetrySettings = w.RetrySettings
+
+	s.RetrySettings.RetryCodes = make(map[codes.Code]bool, len(w.RetrySettings.RetryCodes))
+	for key, value := range w.RetrySettings.RetryCodes {
+		s.RetrySettings.RetryCodes[key] = value
+	}
+}
+
+type withRetryCodes []codes.Code
+
+func (w withRetryCodes) Resolve(s *CallSettings) {
+	s.RetrySettings.RetryCodes = make(map[codes.Code]bool)
+	for _, code := range w {
+		s.RetrySettings.RetryCodes[code] = true
+	}
+}
+
+// WithRetryCodes sets a list of Google API canonical error codes upon which a
+// retry should be attempted.
+func WithRetryCodes(retryCodes []codes.Code) CallOption {
+	return withRetryCodes(retryCodes)
+}
+
+type withDelayTimeoutSettings MultipliableDuration
+
+func (w withDelayTimeoutSettings) Resolve(s *CallSettings) {
+	s.RetrySettings.BackoffSettings.DelayTimeoutSettings = MultipliableDuration(w)
+}
+
+// WithDelayTimeoutSettings specifies:
+// - The initial delay time, in milliseconds, between the completion of
+//   the first failed request and the initiation of the first retrying
+//   request.
+// - The multiplier by which to increase the delay time between the
+//   completion of failed requests, and the initiation of the subsequent
+//   retrying request.
+// - The maximum delay time, in milliseconds, between requests. When this
+//   value is reached, `RetryDelayMultiplier` will no longer be used to
+//   increase delay time.
+func WithDelayTimeoutSettings(initial time.Duration, max time.Duration, multiplier float64) CallOption {
+	return withDelayTimeoutSettings(MultipliableDuration{initial, max, multiplier})
+}

vendor/cloud.google.com/go/bigtable/internal/gax/invoke.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2015 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This is ia snapshot from github.com/googleapis/gax-go with minor modifications.
+package gax
+
+import (
+	"math/rand"
+	"time"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"log"
+	"os"
+)
+
+var logger *log.Logger = log.New(os.Stderr, "", log.LstdFlags)
+
+// A user defined call stub.
+type APICall func(context.Context) error
+
+// scaleDuration returns the product of a and mult.
+func scaleDuration(a time.Duration, mult float64) time.Duration {
+	ns := float64(a) * mult
+	return time.Duration(ns)
+}
+
+// invokeWithRetry calls stub using an exponential backoff retry mechanism
+// based on the values provided in callSettings.
+func invokeWithRetry(ctx context.Context, stub APICall, callSettings CallSettings) error {
+	retrySettings := callSettings.RetrySettings
+	backoffSettings := callSettings.RetrySettings.BackoffSettings
+	delay := backoffSettings.DelayTimeoutSettings.Initial
+	for {
+		// If the deadline is exceeded...
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		err := stub(ctx)
+		code := grpc.Code(err)
+		if code == codes.OK {
+			return nil
+		}
+
+		if !retrySettings.RetryCodes[code] {
+			return err
+		}
+
+		// Sleep a random amount up to the current delay
+		d := time.Duration(rand.Int63n(int64(delay)))
+		delayCtx, _ := context.WithTimeout(ctx, delay)
+		logger.Printf("Retryable error: %v, retrying in %v", err, d)
+		<-delayCtx.Done()
+
+		delay = scaleDuration(delay, backoffSettings.DelayTimeoutSettings.Multiplier)
+		if delay > backoffSettings.DelayTimeoutSettings.Max {
+			delay = backoffSettings.DelayTimeoutSettings.Max
+		}
+	}
+}
+
+// Invoke calls stub with a child of context modified by the specified options.
+func Invoke(ctx context.Context, stub APICall, opts ...CallOption) error {
+	settings := &CallSettings{}
+	callOptions(opts).Resolve(settings)
+	if len(settings.RetrySettings.RetryCodes) > 0 {
+		return invokeWithRetry(ctx, stub, *settings)
+	}
+	return stub(ctx)
+}

vendor/cloud.google.com/go/bigtable/internal/gax/invoke_test.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2015 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package gax
+
+import (
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+)
+
+func TestRandomizedDelays(t *testing.T) {
+	max := 200 * time.Millisecond
+	settings := []CallOption{
+		WithRetryCodes([]codes.Code{codes.Unavailable, codes.DeadlineExceeded}),
+		WithDelayTimeoutSettings(10*time.Millisecond, max, 1.5),
+	}
+
+	deadline := time.Now().Add(1 * time.Second)
+	ctx, _ := context.WithDeadline(context.Background(), deadline)
+	var invokeTime time.Time
+	Invoke(ctx, func(childCtx context.Context) error {
+		// Keep failing, make sure we never slept more than max (plus a fudge factor)
+		if !invokeTime.IsZero() {
+			if got, want := time.Since(invokeTime), max; got > (want + 20*time.Millisecond) {
+				t.Logf("Slept too long. Got: %v, want: %v", got, max)
+			}
+		}
+		invokeTime = time.Now()
+		// Workaround for `go vet`: https://github.com/grpc/grpc-go/issues/90
+		errf := grpc.Errorf
+		return errf(codes.Unavailable, "")
+	}, settings...)
+}

vendor/cloud.google.com/go/bigtable/internal/option/option.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2015 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package option contains common code for dealing with client options.
+package option
+
+import (
+	"fmt"
+	"os"
+
+	"google.golang.org/api/option"
+	"google.golang.org/grpc"
+)
+
+// DefaultClientOptions returns the default client options to use for the
+// client's gRPC connection.
+func DefaultClientOptions(endpoint, scope, userAgent string) ([]option.ClientOption, error) {
+	var o []option.ClientOption
+	// Check the environment variables for the bigtable emulator.
+	// Dial it directly and don't pass any credentials.
+	if addr := os.Getenv("BIGTABLE_EMULATOR_HOST"); addr != "" {
+		conn, err := grpc.Dial(addr, grpc.WithInsecure())
+		if err != nil {
+			return nil, fmt.Errorf("emulator grpc.Dial: %v", err)
+		}
+		o = []option.ClientOption{option.WithGRPCConn(conn)}
+	} else {
+		o = []option.ClientOption{
+			option.WithEndpoint(endpoint),
+			option.WithScopes(scope),
+			option.WithUserAgent(userAgent),
+		}
+	}
+	return o, nil
+}

vendor/cloud.google.com/go/bigtable/internal/stat/stats.go

@@ -0,0 +1,144 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package stat
+
+import (
+	"bytes"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strconv"
+	"text/tabwriter"
+	"time"
+)
+
+type byDuration []time.Duration
+
+func (data byDuration) Len() int           { return len(data) }
+func (data byDuration) Swap(i, j int)      { data[i], data[j] = data[j], data[i] }
+func (data byDuration) Less(i, j int) bool { return data[i] < data[j] }
+
+// quantile returns a value representing the kth of q quantiles.
+// May alter the order of data.
+func quantile(data []time.Duration, k, q int) (quantile time.Duration, ok bool) {
+	if len(data) < 1 {
+		return 0, false
+	}
+	if k > q {
+		return 0, false
+	}
+	if k < 0 || q < 1 {
+		return 0, false
+	}
+
+	sort.Sort(byDuration(data))
+
+	if k == 0 {
+		return data[0], true
+	}
+	if k == q {
+		return data[len(data)-1], true
+	}
+
+	bucketSize := float64(len(data)-1) / float64(q)
+	i := float64(k) * bucketSize
+
+	lower := int(math.Trunc(i))
+	var upper int
+	if i > float64(lower) && lower+1 < len(data) {
+		// If the quantile lies between two elements
+		upper = lower + 1
+	} else {
+		upper = lower
+	}
+	weightUpper := i - float64(lower)
+	weightLower := 1 - weightUpper
+	return time.Duration(weightLower*float64(data[lower]) + weightUpper*float64(data[upper])), true
+}
+
+type Aggregate struct {
+	Name               string
+	Count, Errors      int
+	Min, Median, Max   time.Duration
+	P75, P90, P95, P99 time.Duration // percentiles
+}
+
+// NewAggregate constructs an aggregate from latencies. Returns nil if latencies does not contain aggregateable data.
+func NewAggregate(name string, latencies []time.Duration, errorCount int) *Aggregate {
+	agg := Aggregate{Name: name, Count: len(latencies), Errors: errorCount}
+
+	if len(latencies) == 0 {
+		return nil
+	}
+	var ok bool
+	if agg.Min, ok = quantile(latencies, 0, 2); !ok {
+		return nil
+	}
+	if agg.Median, ok = quantile(latencies, 1, 2); !ok {
+		return nil
+	}
+	if agg.Max, ok = quantile(latencies, 2, 2); !ok {
+		return nil
+	}
+	if agg.P75, ok = quantile(latencies, 75, 100); !ok {
+		return nil
+	}
+	if agg.P90, ok = quantile(latencies, 90, 100); !ok {
+		return nil
+	}
+	if agg.P95, ok = quantile(latencies, 95, 100); !ok {
+		return nil
+	}
+	if agg.P99, ok = quantile(latencies, 99, 100); !ok {
+		return nil
+	}
+	return &agg
+}
+
+func (agg *Aggregate) String() string {
+	if agg == nil {
+		return "no data"
+	}
+	var buf bytes.Buffer
+	tw := tabwriter.NewWriter(&buf, 0, 0, 1, ' ', 0) // one-space padding
+	fmt.Fprintf(tw, "min:\t%v\nmedian:\t%v\nmax:\t%v\n95th percentile:\t%v\n99th percentile:\t%v\n",
+		agg.Min, agg.Median, agg.Max, agg.P95, agg.P99)
+	tw.Flush()
+	return buf.String()
+}
+
+// WriteCSV writes a csv file to the given Writer,
+// with a header row and one row per aggregate.
+func WriteCSV(aggs []*Aggregate, iow io.Writer) error {
+	w := csv.NewWriter(iow)
+	defer w.Flush()
+	err := w.Write([]string{"name", "count", "errors", "min", "median", "max", "p75", "p90", "p95", "p99"})
+	if err != nil {
+		return err
+	}
+	for _, agg := range aggs {
+		err = w.Write([]string{
+			agg.Name, strconv.Itoa(agg.Count), strconv.Itoa(agg.Errors),
+			agg.Min.String(), agg.Median.String(), agg.Max.String(),
+			agg.P75.String(), agg.P90.String(), agg.P95.String(), agg.P99.String(),
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}