Update dependenciess

Exclude minio-go for now (pin to 3.x.y).
This commit is contained in:
Alexander Neumann
2017-12-03 21:01:25 +01:00
parent 9d0f13c4c0
commit 946c8399e2
2985 changed files with 1008107 additions and 118934 deletions

View File

@@ -21,12 +21,6 @@ import (
// ExtractConfig holds the configuration for an extract job.
type ExtractConfig struct {
// JobID is the ID to use for the job. If empty, a random job ID will be generated.
JobID string
// If AddJobIDSuffix is true, then a random string will be appended to JobID.
AddJobIDSuffix bool
// Src is the table from which data will be extracted.
Src *Table
@@ -35,10 +29,52 @@ type ExtractConfig struct {
// DisableHeader disables the printing of a header row in exported data.
DisableHeader bool
// The labels associated with this job.
Labels map[string]string
}
func (e *ExtractConfig) toBQ() *bq.JobConfiguration {
var printHeader *bool
if e.DisableHeader {
f := false
printHeader = &f
}
return &bq.JobConfiguration{
Labels: e.Labels,
Extract: &bq.JobConfigurationExtract{
DestinationUris: append([]string{}, e.Dst.URIs...),
Compression: string(e.Dst.Compression),
DestinationFormat: string(e.Dst.DestinationFormat),
FieldDelimiter: e.Dst.FieldDelimiter,
SourceTable: e.Src.toBQ(),
PrintHeader: printHeader,
},
}
}
func bqToExtractConfig(q *bq.JobConfiguration, c *Client) *ExtractConfig {
qe := q.Extract
return &ExtractConfig{
Labels: q.Labels,
Dst: &GCSReference{
URIs: qe.DestinationUris,
Compression: Compression(qe.Compression),
DestinationFormat: DataFormat(qe.DestinationFormat),
FileConfig: FileConfig{
CSVOptions: CSVOptions{
FieldDelimiter: qe.FieldDelimiter,
},
},
},
DisableHeader: qe.PrintHeader != nil && !*qe.PrintHeader,
Src: bqToTable(qe.SourceTable, c),
}
}
// An Extractor extracts data from a BigQuery table into Google Cloud Storage.
type Extractor struct {
JobIDConfig
ExtractConfig
c *Client
}
@@ -58,23 +94,12 @@ func (t *Table) ExtractorTo(dst *GCSReference) *Extractor {
// Run initiates an extract job.
func (e *Extractor) Run(ctx context.Context) (*Job, error) {
var printHeader *bool
if e.DisableHeader {
f := false
printHeader = &f
}
job := &bq.Job{
JobReference: createJobRef(e.JobID, e.AddJobIDSuffix, e.c.projectID),
Configuration: &bq.JobConfiguration{
Extract: &bq.JobConfigurationExtract{
DestinationUris: append([]string{}, e.Dst.uris...),
Compression: string(e.Dst.Compression),
DestinationFormat: string(e.Dst.DestinationFormat),
FieldDelimiter: e.Dst.FieldDelimiter,
SourceTable: e.Src.tableRefProto(),
PrintHeader: printHeader,
},
},
}
return e.c.insertJob(ctx, &insertJobConf{job: job})
return e.c.insertJob(ctx, e.newJob(), nil)
}
func (e *Extractor) newJob() *bq.Job {
return &bq.Job{
JobReference: e.JobIDConfig.createJobRef(e.c.projectID),
Configuration: e.ExtractConfig.toBQ(),
}
}