mirror of
https://github.com/restic/restic.git
synced 2025-10-29 14:03:06 +00:00
Update dependenciess
Exclude minio-go for now (pin to 3.x.y).
This commit is contained in:
77
vendor/cloud.google.com/go/bigquery/load.go
generated
vendored
77
vendor/cloud.google.com/go/bigquery/load.go
generated
vendored
@@ -15,18 +15,14 @@
|
||||
package bigquery
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
bq "google.golang.org/api/bigquery/v2"
|
||||
)
|
||||
|
||||
// LoadConfig holds the configuration for a load job.
|
||||
type LoadConfig struct {
|
||||
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
|
||||
JobID string
|
||||
|
||||
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
|
||||
AddJobIDSuffix bool
|
||||
|
||||
// Src is the source from which data will be loaded.
|
||||
Src LoadSource
|
||||
|
||||
@@ -40,10 +36,53 @@ type LoadConfig struct {
|
||||
// WriteDisposition specifies how existing data in the destination table is treated.
|
||||
// The default is WriteAppend.
|
||||
WriteDisposition TableWriteDisposition
|
||||
|
||||
// The labels associated with this job.
|
||||
Labels map[string]string
|
||||
|
||||
// If non-nil, the destination table is partitioned by time.
|
||||
TimePartitioning *TimePartitioning
|
||||
}
|
||||
|
||||
func (l *LoadConfig) toBQ() (*bq.JobConfiguration, io.Reader) {
|
||||
config := &bq.JobConfiguration{
|
||||
Labels: l.Labels,
|
||||
Load: &bq.JobConfigurationLoad{
|
||||
CreateDisposition: string(l.CreateDisposition),
|
||||
WriteDisposition: string(l.WriteDisposition),
|
||||
DestinationTable: l.Dst.toBQ(),
|
||||
TimePartitioning: l.TimePartitioning.toBQ(),
|
||||
},
|
||||
}
|
||||
media := l.Src.populateLoadConfig(config.Load)
|
||||
return config, media
|
||||
}
|
||||
|
||||
func bqToLoadConfig(q *bq.JobConfiguration, c *Client) *LoadConfig {
|
||||
lc := &LoadConfig{
|
||||
Labels: q.Labels,
|
||||
CreateDisposition: TableCreateDisposition(q.Load.CreateDisposition),
|
||||
WriteDisposition: TableWriteDisposition(q.Load.WriteDisposition),
|
||||
Dst: bqToTable(q.Load.DestinationTable, c),
|
||||
TimePartitioning: bqToTimePartitioning(q.Load.TimePartitioning),
|
||||
}
|
||||
var fc *FileConfig
|
||||
if len(q.Load.SourceUris) == 0 {
|
||||
s := NewReaderSource(nil)
|
||||
fc = &s.FileConfig
|
||||
lc.Src = s
|
||||
} else {
|
||||
s := NewGCSReference(q.Load.SourceUris...)
|
||||
fc = &s.FileConfig
|
||||
lc.Src = s
|
||||
}
|
||||
bqPopulateFileConfig(q.Load, fc)
|
||||
return lc
|
||||
}
|
||||
|
||||
// A Loader loads data from Google Cloud Storage into a BigQuery table.
|
||||
type Loader struct {
|
||||
JobIDConfig
|
||||
LoadConfig
|
||||
c *Client
|
||||
}
|
||||
@@ -54,7 +93,8 @@ type Loader struct {
|
||||
// This package defines two LoadSources: GCSReference, for Google Cloud Storage
|
||||
// objects, and ReaderSource, for data read from an io.Reader.
|
||||
type LoadSource interface {
|
||||
populateInsertJobConfForLoad(conf *insertJobConf)
|
||||
// populates config, returns media
|
||||
populateLoadConfig(*bq.JobConfigurationLoad) io.Reader
|
||||
}
|
||||
|
||||
// LoaderFrom returns a Loader which can be used to load data into a BigQuery table.
|
||||
@@ -73,17 +113,14 @@ func (t *Table) LoaderFrom(src LoadSource) *Loader {
|
||||
|
||||
// Run initiates a load job.
|
||||
func (l *Loader) Run(ctx context.Context) (*Job, error) {
|
||||
job := &bq.Job{
|
||||
JobReference: createJobRef(l.JobID, l.AddJobIDSuffix, l.c.projectID),
|
||||
Configuration: &bq.JobConfiguration{
|
||||
Load: &bq.JobConfigurationLoad{
|
||||
CreateDisposition: string(l.CreateDisposition),
|
||||
WriteDisposition: string(l.WriteDisposition),
|
||||
},
|
||||
},
|
||||
}
|
||||
conf := &insertJobConf{job: job}
|
||||
l.Src.populateInsertJobConfForLoad(conf)
|
||||
job.Configuration.Load.DestinationTable = l.Dst.tableRefProto()
|
||||
return l.c.insertJob(ctx, conf)
|
||||
job, media := l.newJob()
|
||||
return l.c.insertJob(ctx, job, media)
|
||||
}
|
||||
|
||||
func (l *Loader) newJob() (*bq.Job, io.Reader) {
|
||||
config, media := l.LoadConfig.toBQ()
|
||||
return &bq.Job{
|
||||
JobReference: l.JobIDConfig.createJobRef(l.c.projectID),
|
||||
Configuration: config,
|
||||
}, media
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user