vendor: add github.com/diegomarangoni/gcscache

also include all transitive dependencies of gcscache, which seems to
bring in the entire world of Google Cloud Platform.
This commit is contained in:
Will Norris 2017-09-01 20:44:01 +00:00
parent fcb6465a29
commit 0fe3656939
251 changed files with 90555 additions and 0 deletions

252
vendor/cloud.google.com/go/storage/acl.go generated vendored Normal file
View file

@ -0,0 +1,252 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"net/http"
"reflect"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
// ACLRole is the level of access to grant.
type ACLRole string
const (
RoleOwner ACLRole = "OWNER"
RoleReader ACLRole = "READER"
RoleWriter ACLRole = "WRITER"
)
// ACLEntity refers to a user or group.
// They are sometimes referred to as grantees.
//
// It could be in the form of:
// "user-<userId>", "user-<email>", "group-<groupId>", "group-<email>",
// "domain-<domain>" and "project-team-<projectId>".
//
// Or one of the predefined constants: AllUsers, AllAuthenticatedUsers.
type ACLEntity string
const (
AllUsers ACLEntity = "allUsers"
AllAuthenticatedUsers ACLEntity = "allAuthenticatedUsers"
)
// ACLRule represents a grant for a role to an entity (user, group or team) for a Google Cloud Storage object or bucket.
type ACLRule struct {
Entity ACLEntity
Role ACLRole
}
// ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object.
type ACLHandle struct {
c *Client
bucket string
object string
isDefault bool
userProject string // for requester-pays buckets
}
// Delete permanently deletes the ACL entry for the given entity.
func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error {
if a.object != "" {
return a.objectDelete(ctx, entity)
}
if a.isDefault {
return a.bucketDefaultDelete(ctx, entity)
}
return a.bucketDelete(ctx, entity)
}
// Set sets the permission level for the given entity.
func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error {
if a.object != "" {
return a.objectSet(ctx, entity, role, false)
}
if a.isDefault {
return a.objectSet(ctx, entity, role, true)
}
return a.bucketSet(ctx, entity, role)
}
// List retrieves ACL entries.
func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) {
if a.object != "" {
return a.objectList(ctx)
}
if a.isDefault {
return a.bucketDefaultList(ctx)
}
return a.bucketList(ctx)
}
func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err)
}
return toACLRules(acls.Items), nil
}
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.BucketAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.List(a.bucket)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err)
}
r := make([]ACLRule, len(acls.Items))
for i, v := range acls.Items {
r[i].Entity = ACLEntity(v.Entity)
r[i].Role = ACLRole(v.Role)
}
return r, nil
}
func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
acl := &raw.BucketAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
err := runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
a.configureCall(req, ctx)
_, err := req.Do()
return err
})
if err != nil {
return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
}
return nil
}
func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
if err != nil {
return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err)
}
return toACLRules(acls.Items), nil
}
func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error {
type setRequest interface {
Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error)
Header() http.Header
}
acl := &raw.ObjectAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
var req setRequest
if isBucketDefault {
req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl)
} else {
req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
}
a.configureCall(req, ctx)
err := runWithRetry(ctx, func() error {
_, err := req.Do()
return err
})
if err != nil {
if isBucketDefault {
return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
} else {
return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err)
}
}
return nil
}
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err)
}
return nil
}
func (a *ACLHandle) configureCall(call interface {
Header() http.Header
}, ctx context.Context) {
vc := reflect.ValueOf(call)
vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)})
if a.userProject != "" {
vc.MethodByName("UserProject").Call([]reflect.Value{reflect.ValueOf(a.userProject)})
}
setClientHeader(call.Header())
}
func toACLRules(items []*raw.ObjectAccessControl) []ACLRule {
r := make([]ACLRule, 0, len(items))
for _, item := range items {
r = append(r, ACLRule{Entity: ACLEntity(item.Entity), Role: ACLRole(item.Role)})
}
return r
}

758
vendor/cloud.google.com/go/storage/bucket.go generated vendored Normal file
View file

@ -0,0 +1,758 @@
// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"net/http"
"reflect"
"time"
"cloud.google.com/go/internal/optional"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
raw "google.golang.org/api/storage/v1"
)
// BucketHandle provides operations on a Google Cloud Storage bucket.
// Use Client.Bucket to get a handle.
type BucketHandle struct {
c *Client
name string
acl ACLHandle
defaultObjectACL ACLHandle
conds *BucketConditions
userProject string // project for requester-pays buckets
}
// Bucket returns a BucketHandle, which provides operations on the named bucket.
// This call does not perform any network operations.
//
// The supplied name must contain only lowercase letters, numbers, dashes,
// underscores, and dots. The full specification for valid bucket names can be
// found at:
// https://cloud.google.com/storage/docs/bucket-naming
func (c *Client) Bucket(name string) *BucketHandle {
return &BucketHandle{
c: c,
name: name,
acl: ACLHandle{
c: c,
bucket: name,
},
defaultObjectACL: ACLHandle{
c: c,
bucket: name,
isDefault: true,
},
}
}
// Create creates the Bucket in the project.
// If attrs is nil the API defaults will be used.
func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error {
var bkt *raw.Bucket
if attrs != nil {
bkt = attrs.toRawBucket()
} else {
bkt = &raw.Bucket{}
}
bkt.Name = b.name
// If there is lifecycle information but no location, explicitly set
// the location. This is a GCS quirk/bug.
if bkt.Location == "" && bkt.Lifecycle != nil {
bkt.Location = "US"
}
req := b.c.raw.Buckets.Insert(projectID, bkt)
setClientHeader(req.Header())
return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err })
}
// Delete deletes the Bucket.
func (b *BucketHandle) Delete(ctx context.Context) error {
req, err := b.newDeleteCall()
if err != nil {
return err
}
return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
}
func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
req := b.c.raw.Buckets.Delete(b.name)
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// ACL returns an ACLHandle, which provides access to the bucket's access control list.
// This controls who can list, create or overwrite the objects in a bucket.
// This call does not perform any network operations.
func (b *BucketHandle) ACL() *ACLHandle {
return &b.acl
}
// DefaultObjectACL returns an ACLHandle, which provides access to the bucket's default object ACLs.
// These ACLs are applied to newly created objects in this bucket that do not have a defined ACL.
// This call does not perform any network operations.
func (b *BucketHandle) DefaultObjectACL() *ACLHandle {
return &b.defaultObjectACL
}
// Object returns an ObjectHandle, which provides operations on the named object.
// This call does not perform any network operations.
//
// name must consist entirely of valid UTF-8-encoded runes. The full specification
// for valid object names can be found at:
// https://cloud.google.com/storage/docs/bucket-naming
func (b *BucketHandle) Object(name string) *ObjectHandle {
return &ObjectHandle{
c: b.c,
bucket: b.name,
object: name,
acl: ACLHandle{
c: b.c,
bucket: b.name,
object: name,
userProject: b.userProject,
},
gen: -1,
userProject: b.userProject,
}
}
// Attrs returns the metadata for the bucket.
func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) {
req, err := b.newGetCall()
if err != nil {
return nil, err
}
var resp *raw.Bucket
err = runWithRetry(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
})
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
return nil, ErrBucketNotExist
}
if err != nil {
return nil, err
}
return newBucket(resp), nil
}
func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) {
req := b.c.raw.Buckets.Get(b.name).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) {
req, err := b.newPatchCall(&uattrs)
if err != nil {
return nil, err
}
// TODO(jba): retry iff metagen is set?
rb, err := req.Context(ctx).Do()
if err != nil {
return nil, err
}
return newBucket(rb), nil
}
func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
rb := uattrs.toRawBucket()
req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full")
setClientHeader(req.Header())
if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil {
return nil, err
}
if b.userProject != "" {
req.UserProject(b.userProject)
}
return req, nil
}
// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
type BucketAttrs struct {
// Name is the name of the bucket.
Name string
// ACL is the list of access control rules on the bucket.
ACL []ACLRule
// DefaultObjectACL is the list of access controls to
// apply to new objects when no object ACL is provided.
DefaultObjectACL []ACLRule
// Location is the location of the bucket. It defaults to "US".
Location string
// MetaGeneration is the metadata generation of the bucket.
MetaGeneration int64
// StorageClass is the default storage class of the bucket. This defines
// how objects in the bucket are stored and determines the SLA
// and the cost of storage. Typical values are "MULTI_REGIONAL",
// "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" and
// "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD", which
// is equivalent to "MULTI_REGIONAL" or "REGIONAL" depending on
// the bucket's location settings.
StorageClass string
// Created is the creation time of the bucket.
Created time.Time
// VersioningEnabled reports whether this bucket has versioning enabled.
// This field is read-only.
VersioningEnabled bool
// Labels are the bucket's labels.
Labels map[string]string
// RequesterPays reports whether the bucket is a Requester Pays bucket.
RequesterPays bool
// Lifecycle is the lifecycle configuration for objects in the bucket.
Lifecycle Lifecycle
}
// Lifecycle is the lifecycle configuration for objects in the bucket.
type Lifecycle struct {
Rules []LifecycleRule
}
const (
// RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule.
rfc3339Date = "2006-01-02"
// DeleteAction is a lifecycle action that deletes a live and/or archived
// objects. Takes precendence over SetStorageClass actions.
DeleteAction = "Delete"
// SetStorageClassAction changes the storage class of live and/or archived
// objects.
SetStorageClassAction = "SetStorageClass"
)
// LifecycleRule is a lifecycle configuration rule.
//
// When all the configured conditions are met by an object in the bucket, the
// configured action will automatically be taken on that object.
type LifecycleRule struct {
// Action is the action to take when all of the associated conditions are
// met.
Action LifecycleAction
// Condition is the set of conditions that must be met for the associated
// action to be taken.
Condition LifecycleCondition
}
// LifecycleAction is a lifecycle configuration action.
type LifecycleAction struct {
// Type is the type of action to take on matching objects.
//
// Acceptable values are "Delete" to delete matching objects and
// "SetStorageClass" to set the storage class defined in StorageClass on
// matching objects.
Type string
// StorageClass is the storage class to set on matching objects if the Action
// is "SetStorageClass".
StorageClass string
}
// Liveness specifies whether the object is live or not.
type Liveness int
const (
// LiveAndArchived includes both live and archived objects.
LiveAndArchived Liveness = iota
// Live specifies that the object is still live.
Live
// Archived specifies that the object is archived.
Archived
)
// LifecycleCondition is a set of conditions used to match objects and take an
// action automatically.
//
// All configured conditions must be met for the associated action to be taken.
type LifecycleCondition struct {
// AgeInDays is the age of the object in days.
AgeInDays int64
// CreatedBefore is the time the object was created.
//
// This condition is satisfied when an object is created before midnight of
// the specified date in UTC.
CreatedBefore time.Time
// Liveness specifies the object's liveness. Relevant only for versioned objects
Liveness Liveness
// MatchesStorageClasses is the condition matching the object's storage
// class.
//
// Values include "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE",
// "STANDARD", and "DURABLE_REDUCED_AVAILABILITY".
MatchesStorageClasses []string
// NumNewerVersions is the condition matching objects with a number of newer versions.
//
// If the value is N, this condition is satisfied when there are at least N
// versions (including the live version) newer than this version of the
// object.
NumNewerVersions int64
}
func newBucket(b *raw.Bucket) *BucketAttrs {
if b == nil {
return nil
}
bucket := &BucketAttrs{
Name: b.Name,
Location: b.Location,
MetaGeneration: b.Metageneration,
StorageClass: b.StorageClass,
Created: convertTime(b.TimeCreated),
VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled,
Labels: b.Labels,
RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
Lifecycle: toLifecycle(b.Lifecycle),
}
acl := make([]ACLRule, len(b.Acl))
for i, rule := range b.Acl {
acl[i] = ACLRule{
Entity: ACLEntity(rule.Entity),
Role: ACLRole(rule.Role),
}
}
bucket.ACL = acl
objACL := make([]ACLRule, len(b.DefaultObjectAcl))
for i, rule := range b.DefaultObjectAcl {
objACL[i] = ACLRule{
Entity: ACLEntity(rule.Entity),
Role: ACLRole(rule.Role),
}
}
bucket.DefaultObjectACL = objACL
return bucket
}
// toRawBucket copies the editable attribute from b to the raw library's Bucket type.
func (b *BucketAttrs) toRawBucket() *raw.Bucket {
var acl []*raw.BucketAccessControl
if len(b.ACL) > 0 {
acl = make([]*raw.BucketAccessControl, len(b.ACL))
for i, rule := range b.ACL {
acl[i] = &raw.BucketAccessControl{
Entity: string(rule.Entity),
Role: string(rule.Role),
}
}
}
dACL := toRawObjectACL(b.DefaultObjectACL)
// Copy label map.
var labels map[string]string
if len(b.Labels) > 0 {
labels = make(map[string]string, len(b.Labels))
for k, v := range b.Labels {
labels[k] = v
}
}
// Ignore VersioningEnabled if it is false. This is OK because
// we only call this method when creating a bucket, and by default
// new buckets have versioning off.
var v *raw.BucketVersioning
if b.VersioningEnabled {
v = &raw.BucketVersioning{Enabled: true}
}
var bb *raw.BucketBilling
if b.RequesterPays {
bb = &raw.BucketBilling{RequesterPays: true}
}
return &raw.Bucket{
Name: b.Name,
DefaultObjectAcl: dACL,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: acl,
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toRawLifecycle(b.Lifecycle),
}
}
type BucketAttrsToUpdate struct {
// VersioningEnabled, if set, updates whether the bucket uses versioning.
VersioningEnabled optional.Bool
// RequesterPays, if set, updates whether the bucket is a Requester Pays bucket.
RequesterPays optional.Bool
setLabels map[string]string
deleteLabels map[string]bool
}
// SetLabel causes a label to be added or modified when ua is used
// in a call to Bucket.Update.
func (ua *BucketAttrsToUpdate) SetLabel(name, value string) {
if ua.setLabels == nil {
ua.setLabels = map[string]string{}
}
ua.setLabels[name] = value
}
// DeleteLabel causes a label to be deleted when ua is used in a
// call to Bucket.Update.
func (ua *BucketAttrsToUpdate) DeleteLabel(name string) {
if ua.deleteLabels == nil {
ua.deleteLabels = map[string]bool{}
}
ua.deleteLabels[name] = true
}
func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
rb := &raw.Bucket{}
if ua.VersioningEnabled != nil {
rb.Versioning = &raw.BucketVersioning{
Enabled: optional.ToBool(ua.VersioningEnabled),
ForceSendFields: []string{"Enabled"},
}
}
if ua.RequesterPays != nil {
rb.Billing = &raw.BucketBilling{
RequesterPays: optional.ToBool(ua.RequesterPays),
ForceSendFields: []string{"RequesterPays"},
}
}
if ua.setLabels != nil || ua.deleteLabels != nil {
rb.Labels = map[string]string{}
for k, v := range ua.setLabels {
rb.Labels[k] = v
}
if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 {
rb.ForceSendFields = append(rb.ForceSendFields, "Labels")
}
for l := range ua.deleteLabels {
rb.NullFields = append(rb.NullFields, "Labels."+l)
}
}
return rb
}
// If returns a new BucketHandle that applies a set of preconditions.
// Preconditions already set on the BucketHandle are ignored.
// Operations on the new handle will only occur if the preconditions are
// satisfied. The only valid preconditions for buckets are MetagenerationMatch
// and MetagenerationNotMatch.
func (b *BucketHandle) If(conds BucketConditions) *BucketHandle {
b2 := *b
b2.conds = &conds
return &b2
}
// BucketConditions constrain bucket methods to act on specific metagenerations.
//
// The zero value is an empty set of constraints.
type BucketConditions struct {
// MetagenerationMatch specifies that the bucket must have the given
// metageneration for the operation to occur.
// If MetagenerationMatch is zero, it has no effect.
MetagenerationMatch int64
// MetagenerationNotMatch specifies that the bucket must not have the given
// metageneration for the operation to occur.
// If MetagenerationNotMatch is zero, it has no effect.
MetagenerationNotMatch int64
}
func (c *BucketConditions) validate(method string) error {
if *c == (BucketConditions{}) {
return fmt.Errorf("storage: %s: empty conditions", method)
}
if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 {
return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
}
return nil
}
// UserProject returns a new BucketHandle that passes the project ID as the user
// project for all subsequent calls. A user project is required for all operations
// on requester-pays buckets.
func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
b2 := *b
b2.userProject = projectID
b2.acl.userProject = projectID
b2.defaultObjectACL.userProject = projectID
return &b2
}
// applyBucketConds modifies the provided call using the conditions in conds.
// call is something that quacks like a *raw.WhateverCall.
func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
if conds == nil {
return nil
}
if err := conds.validate(method); err != nil {
return err
}
cval := reflect.ValueOf(call)
switch {
case conds.MetagenerationMatch != 0:
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
}
case conds.MetagenerationNotMatch != 0:
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
}
}
return nil
}
func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
var rl raw.BucketLifecycle
if len(l.Rules) == 0 {
return nil
}
for _, r := range l.Rules {
rr := &raw.BucketLifecycleRule{
Action: &raw.BucketLifecycleRuleAction{
Type: r.Action.Type,
StorageClass: r.Action.StorageClass,
},
Condition: &raw.BucketLifecycleRuleCondition{
Age: r.Condition.AgeInDays,
MatchesStorageClass: r.Condition.MatchesStorageClasses,
NumNewerVersions: r.Condition.NumNewerVersions,
},
}
switch r.Condition.Liveness {
case LiveAndArchived:
rr.Condition.IsLive = nil
case Live:
rr.Condition.IsLive = googleapi.Bool(true)
case Archived:
rr.Condition.IsLive = googleapi.Bool(false)
}
if !r.Condition.CreatedBefore.IsZero() {
rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date)
}
rl.Rule = append(rl.Rule, rr)
}
return &rl
}
func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
var l Lifecycle
if rl == nil {
return l
}
for _, rr := range rl.Rule {
r := LifecycleRule{
Action: LifecycleAction{
Type: rr.Action.Type,
StorageClass: rr.Action.StorageClass,
},
Condition: LifecycleCondition{
AgeInDays: rr.Condition.Age,
MatchesStorageClasses: rr.Condition.MatchesStorageClass,
NumNewerVersions: rr.Condition.NumNewerVersions,
},
}
switch {
case rr.Condition.IsLive == nil:
r.Condition.Liveness = LiveAndArchived
case *rr.Condition.IsLive == true:
r.Condition.Liveness = Live
case *rr.Condition.IsLive == false:
r.Condition.Liveness = Archived
}
if rr.Condition.CreatedBefore != "" {
r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
}
}
return l
}
// Objects returns an iterator over the objects in the bucket that match the Query q.
// If q is nil, no filtering is done.
func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
it := &ObjectIterator{
ctx: ctx,
bucket: b,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
if q != nil {
it.query = *q
}
return it
}
// An ObjectIterator is an iterator over ObjectAttrs.
type ObjectIterator struct {
ctx context.Context
bucket *BucketHandle
query Query
pageInfo *iterator.PageInfo
nextFunc func() error
items []*ObjectAttrs
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
//
// If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will
// have a non-empty Prefix field, and a zero value for all other fields. These
// represent prefixes.
func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.bucket.c.raw.Objects.List(it.bucket.name)
setClientHeader(req.Header())
req.Projection("full")
req.Delimiter(it.query.Delimiter)
req.Prefix(it.query.Prefix)
req.Versions(it.query.Versions)
req.PageToken(pageToken)
if it.bucket.userProject != "" {
req.UserProject(it.bucket.userProject)
}
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Objects
var err error
err = runWithRetry(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
})
if err != nil {
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
err = ErrBucketNotExist
}
return "", err
}
for _, item := range resp.Items {
it.items = append(it.items, newObject(item))
}
for _, prefix := range resp.Prefixes {
it.items = append(it.items, &ObjectAttrs{Prefix: prefix})
}
return resp.NextPageToken, nil
}
// TODO(jbd): Add storage.buckets.update.
// Buckets returns an iterator over the buckets in the project. You may
// optionally set the iterator's Prefix field to restrict the list to buckets
// whose names begin with the prefix. By default, all buckets in the project
// are returned.
func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator {
it := &BucketIterator{
ctx: ctx,
client: c,
projectID: projectID,
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.buckets) },
func() interface{} { b := it.buckets; it.buckets = nil; return b })
return it
}
// A BucketIterator is an iterator over BucketAttrs.
type BucketIterator struct {
// Prefix restricts the iterator to buckets whose names begin with it.
Prefix string
ctx context.Context
client *Client
projectID string
buckets []*BucketAttrs
pageInfo *iterator.PageInfo
nextFunc func() error
}
// Next returns the next result. Its second return value is iterator.Done if
// there are no more results. Once Next returns iterator.Done, all subsequent
// calls will return iterator.Done.
func (it *BucketIterator) Next() (*BucketAttrs, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
b := it.buckets[0]
it.buckets = it.buckets[1:]
return b, nil
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) {
req := it.client.raw.Buckets.List(it.projectID)
setClientHeader(req.Header())
req.Projection("full")
req.Prefix(it.Prefix)
req.PageToken(pageToken)
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
var resp *raw.Buckets
var err error
err = runWithRetry(it.ctx, func() error {
resp, err = req.Context(it.ctx).Do()
return err
})
if err != nil {
return "", err
}
for _, item := range resp.Items {
it.buckets = append(it.buckets, newBucket(item))
}
return resp.NextPageToken, nil
}

201
vendor/cloud.google.com/go/storage/copy.go generated vendored Normal file
View file

@ -0,0 +1,201 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"errors"
"fmt"
"golang.org/x/net/context"
raw "google.golang.org/api/storage/v1"
)
// CopierFrom creates a Copier that can copy src to dst.
// You can immediately call Run on the returned Copier, or
// you can configure it first.
//
// For Requester Pays buckets, the user project of dst is billed, unless it is empty,
// in which case the user project of src is billed.
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
return &Copier{dst: dst, src: src}
}
// A Copier copies a source object to a destination.
type Copier struct {
// ObjectAttrs are optional attributes to set on the destination object.
// Any attributes must be initialized before any calls on the Copier. Nil
// or zero-valued attributes are ignored.
ObjectAttrs
// RewriteToken can be set before calling Run to resume a copy
// operation. After Run returns a non-nil error, RewriteToken will
// have been updated to contain the value needed to resume the copy.
RewriteToken string
// ProgressFunc can be used to monitor the progress of a multi-RPC copy
// operation. If ProgressFunc is not nil and copying requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
// ProgressFunc will be invoked after each call with the number of bytes of
// content copied so far and the total size in bytes of the source object.
//
// ProgressFunc is intended to make upload progress available to the
// application. For example, the implementation of ProgressFunc may update
// a progress bar in the application's UI, or log the result of
// float64(copiedBytes)/float64(totalBytes).
//
// ProgressFunc should return quickly without blocking.
ProgressFunc func(copiedBytes, totalBytes uint64)
dst, src *ObjectHandle
}
// Run performs the copy.
func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
if err := c.src.validate(); err != nil {
return nil, err
}
if err := c.dst.validate(); err != nil {
return nil, err
}
// Convert destination attributes to raw form, omitting the bucket.
// If the bucket is included but name or content-type aren't, the service
// returns a 400 with "Required" as the only message. Omitting the bucket
// does not cause any problems.
rawObject := c.ObjectAttrs.toRawObject("")
for {
res, err := c.callRewrite(ctx, rawObject)
if err != nil {
return nil, err
}
if c.ProgressFunc != nil {
c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize))
}
if res.Done { // Finished successfully.
return newObject(res.Resource), nil
}
}
}
func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) {
call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj)
call.Context(ctx).Projection("full")
if c.RewriteToken != "" {
call.RewriteToken(c.RewriteToken)
}
if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
} else if c.src.userProject != "" {
call.UserProject(c.src.userProject)
}
if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
return nil, err
}
var res *raw.RewriteResponse
var err error
setClientHeader(call.Header())
err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
if err != nil {
return nil, err
}
c.RewriteToken = res.RewriteToken
return res, nil
}
// ComposerFrom creates a Composer that can compose srcs into dst.
// You can immediately call Run on the returned Composer, or you can
// configure it first.
//
// The encryption key for the destination object will be used to decrypt all
// source objects and encrypt the destination object. It is an error
// to specify an encryption key for any of the source objects.
func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
return &Composer{dst: dst, srcs: srcs}
}
// A Composer composes source objects into a destination object.
//
// For Requester Pays buckets, the user project of dst is billed.
type Composer struct {
// ObjectAttrs are optional attributes to set on the destination object.
// Any attributes must be initialized before any calls on the Composer. Nil
// or zero-valued attributes are ignored.
ObjectAttrs
dst *ObjectHandle
srcs []*ObjectHandle
}
// Run performs the compose operation.
func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
if err := c.dst.validate(); err != nil {
return nil, err
}
if len(c.srcs) == 0 {
return nil, errors.New("storage: at least one source object must be specified")
}
req := &raw.ComposeRequest{}
// Compose requires a non-empty Destination, so we always set it,
// even if the caller-provided ObjectAttrs is the zero value.
req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
for _, src := range c.srcs {
if err := src.validate(); err != nil {
return nil, err
}
if src.bucket != c.dst.bucket {
return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket)
}
if src.encryptionKey != nil {
return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
}
srcObj := &raw.ComposeRequestSourceObjects{
Name: src.object,
}
if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
return nil, err
}
req.SourceObjects = append(req.SourceObjects, srcObj)
}
call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
if c.dst.userProject != "" {
call.UserProject(c.dst.userProject)
}
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
var obj *raw.Object
var err error
setClientHeader(call.Header())
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
if err != nil {
return nil, err
}
return newObject(obj), nil
}

161
vendor/cloud.google.com/go/storage/doc.go generated vendored Normal file
View file

@ -0,0 +1,161 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package storage provides an easy way to work with Google Cloud Storage.
Google Cloud Storage stores data in named objects, which are grouped into buckets.
More information about Google Cloud Storage is available at
https://cloud.google.com/storage/docs.
All of the methods of this package use exponential backoff to retry calls
that fail with certain errors, as described in
https://cloud.google.com/storage/docs/exponential-backoff.
Note: This package is in beta. Some backwards-incompatible changes may occur.
Creating a Client
To start working with this package, create a client:
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
// TODO: Handle error.
}
Buckets
A Google Cloud Storage bucket is a collection of objects. To work with a
bucket, make a bucket handle:
bkt := client.Bucket(bucketName)
A handle is a reference to a bucket. You can have a handle even if the
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
call Create on the handle:
if err := bkt.Create(ctx, projectID, nil); err != nil {
// TODO: Handle error.
}
Note that although buckets are associated with projects, bucket names are
global across all projects.
Each bucket has associated metadata, represented in this package by
BucketAttrs. The third argument to BucketHandle.Create allows you to set
the intial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
Attrs:
attrs, err := bkt.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)
Objects
An object holds arbitrary data as a sequence of bytes, like a file. You
refer to objects using a handle, just as with buckets. You can use the
standard Go io.Reader and io.Writer interfaces to read and write
object data:
obj := bkt.Object("data")
// Write something to obj.
// w implements io.Writer.
w := obj.NewWriter(ctx)
// Write some text to obj. This will overwrite whatever is there.
if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
// TODO: Handle error.
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
// TODO: Handle error.
}
// Read it back.
r, err := obj.NewReader(ctx)
if err != nil {
// TODO: Handle error.
}
defer r.Close()
if _, err := io.Copy(os.Stdout, r); err != nil {
// TODO: Handle error.
}
// Prints "This object contains text."
Objects also have attributes, which you can fetch with Attrs:
objAttrs, err := obj.Attrs(ctx)
if err != nil {
// TODO: Handle error.
}
fmt.Printf("object %s has size %d and can be read using %s\n",
objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)
ACLs
Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
ACLRules, each of which specifies the role of a user, group or project. ACLs
are suitable for fine-grained control, but you may prefer using IAM to control
access at the project level (see
https://cloud.google.com/storage/docs/access-control/iam).
To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:
acls, err := obj.ACL().List(ctx)
if err != nil {
// TODO: Handle error.
}
for _, rule := range acls {
fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
}
You can also set and delete ACLs.
Conditions
Every object has a generation and a metageneration. The generation changes
whenever the content changes, and the metageneration changes whenever the
metadata changes. Conditions let you check these values before an operation;
the operation only executes if the conditions match. You can use conditions to
prevent race conditions in read-modify-write operations.
For example, say you've read an object's metadata into objAttrs. Now
you want to write to that object, but only if its contents haven't changed
since you read it. Here is how to express that:
w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
// Proceed with writing as above.
Signed URLs
You can obtain a URL that lets anyone read or write an object for a limited time.
You don't need to create a client to do this. See the documentation of
SignedURL for details.
url, err := storage.SignedURL(bucketName, "shared-object", opts)
if err != nil {
// TODO: Handle error.
}
fmt.Println(url)
Authentication
See examples of authorization and authentication at
https://godoc.org/cloud.google.com/go#pkg-examples.
*/
package storage // import "cloud.google.com/go/storage"

30
vendor/cloud.google.com/go/storage/go110.go generated vendored Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.10
package storage
import "google.golang.org/api/googleapi"
func shouldRetry(err error) bool {
switch e := err.(type) {
case *googleapi.Error:
// Retry on 429 and 5xx, according to
// https://cloud.google.com/storage/docs/exponential-backoff.
return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
default:
return false
}
}

26
vendor/cloud.google.com/go/storage/go17.go generated vendored Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.7
package storage
import (
"context"
"net/http"
)
func withContext(r *http.Request, ctx context.Context) *http.Request {
return r.WithContext(ctx)
}

108
vendor/cloud.google.com/go/storage/iam.go generated vendored Normal file
View file

@ -0,0 +1,108 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"cloud.google.com/go/iam"
"golang.org/x/net/context"
raw "google.golang.org/api/storage/v1"
iampb "google.golang.org/genproto/googleapis/iam/v1"
)
// IAM provides access to IAM access control for the bucket.
func (b *BucketHandle) IAM() *iam.Handle {
return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name)
}
// iamClient implements the iam.client interface.
type iamClient struct {
raw *raw.Service
}
func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) {
req := c.raw.Buckets.GetIamPolicy(resource)
setClientHeader(req.Header())
var rp *raw.Policy
var err error
err = runWithRetry(ctx, func() error {
rp, err = req.Context(ctx).Do()
return err
})
if err != nil {
return nil, err
}
return iamFromStoragePolicy(rp), nil
}
func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error {
rp := iamToStoragePolicy(p)
req := c.raw.Buckets.SetIamPolicy(resource, rp)
setClientHeader(req.Header())
return runWithRetry(ctx, func() error {
_, err := req.Context(ctx).Do()
return err
})
}
func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
req := c.raw.Buckets.TestIamPermissions(resource, perms)
setClientHeader(req.Header())
var res *raw.TestIamPermissionsResponse
var err error
err = runWithRetry(ctx, func() error {
res, err = req.Context(ctx).Do()
return err
})
if err != nil {
return nil, err
}
return res.Permissions, nil
}
func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy {
return &raw.Policy{
Bindings: iamToStorageBindings(ip.Bindings),
Etag: string(ip.Etag),
}
}
func iamToStorageBindings(ibs []*iampb.Binding) []*raw.PolicyBindings {
var rbs []*raw.PolicyBindings
for _, ib := range ibs {
rbs = append(rbs, &raw.PolicyBindings{
Role: ib.Role,
Members: ib.Members,
})
}
return rbs
}
func iamFromStoragePolicy(rp *raw.Policy) *iampb.Policy {
return &iampb.Policy{
Bindings: iamFromStorageBindings(rp.Bindings),
Etag: []byte(rp.Etag),
}
}
func iamFromStorageBindings(rbs []*raw.PolicyBindings) []*iampb.Binding {
var ibs []*iampb.Binding
for _, rb := range rbs {
ibs = append(ibs, &iampb.Binding{
Role: rb.Role,
Members: rb.Members,
})
}
return ibs
}

36
vendor/cloud.google.com/go/storage/invoke.go generated vendored Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"cloud.google.com/go/internal"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
)
// runWithRetry calls the function until it returns nil or a non-retryable error, or
// the context is done.
func runWithRetry(ctx context.Context, call func() error) error {
return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
err = call()
if err == nil {
return true, nil
}
if shouldRetry(err) {
return false, nil
}
return true, err
})
}

40
vendor/cloud.google.com/go/storage/not_go110.go generated vendored Normal file
View file

@ -0,0 +1,40 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.10
package storage
import (
"net/url"
"strings"
"google.golang.org/api/googleapi"
)
func shouldRetry(err error) bool {
switch e := err.(type) {
case *googleapi.Error:
// Retry on 429 and 5xx, according to
// https://cloud.google.com/storage/docs/exponential-backoff.
return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
case *url.Error:
// Retry on REFUSED_STREAM.
// Unfortunately the error type is unexported, so we resort to string
// matching.
return strings.Contains(e.Error(), "REFUSED_STREAM")
default:
return false
}
}

26
vendor/cloud.google.com/go/storage/not_go17.go generated vendored Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.7
package storage
import (
"net/http"
)
func withContext(r *http.Request, _ interface{}) *http.Request {
// In Go 1.6 and below, ignore the context.
return r
}

80
vendor/cloud.google.com/go/storage/reader.go generated vendored Normal file
View file

@ -0,0 +1,80 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"fmt"
"hash/crc32"
"io"
)
var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
// Reader reads a Cloud Storage object.
// It implements io.Reader.
type Reader struct {
body io.ReadCloser
remain, size int64
contentType string
cacheControl string
checkCRC bool // should we check the CRC?
wantCRC uint32 // the CRC32c value the server sent in the header
gotCRC uint32 // running crc
}
// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
return r.body.Close()
}
func (r *Reader) Read(p []byte) (int, error) {
n, err := r.body.Read(p)
if r.remain != -1 {
r.remain -= int64(n)
}
if r.checkCRC {
r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n])
// Check CRC here. It would be natural to check it in Close, but
// everybody defers Close on the assumption that it doesn't return
// anything worth looking at.
if r.remain == 0 && r.gotCRC != r.wantCRC {
return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d",
r.gotCRC, r.wantCRC)
}
}
return n, err
}
// Size returns the size of the object in bytes.
// The returned value is always the same and is not affected by
// calls to Read or Close.
func (r *Reader) Size() int64 {
return r.size
}
// Remain returns the number of bytes left to read, or -1 if unknown.
func (r *Reader) Remain() int64 {
return r.remain
}
// ContentType returns the content type of the object.
func (r *Reader) ContentType() string {
return r.contentType
}
// CacheControl returns the cache control of the object.
func (r *Reader) CacheControl() string {
return r.cacheControl
}

1124
vendor/cloud.google.com/go/storage/storage.go generated vendored Normal file

File diff suppressed because it is too large Load diff

194
vendor/cloud.google.com/go/storage/writer.go generated vendored Normal file
View file

@ -0,0 +1,194 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"encoding/base64"
"errors"
"fmt"
"io"
"unicode/utf8"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
// A Writer writes a Cloud Storage object.
type Writer struct {
// ObjectAttrs are optional attributes to set on the object. Any attributes
// must be initialized before the first Write call. Nil or zero-valued
// attributes are ignored.
ObjectAttrs
// SendCRC specifies whether to transmit a CRC32C field. It should be set
// to true in addition to setting the Writer's CRC32C field, because zero
// is a valid CRC and normally a zero would not be transmitted.
// If a CRC32C is sent, and the data written does not match the checksum,
// the write will be rejected.
SendCRC32C bool
// ChunkSize controls the maximum number of bytes of the object that the
// Writer will attempt to send to the server in a single request. Objects
// smaller than the size will be sent in a single request, while larger
// objects will be split over multiple requests. The size will be rounded up
// to the nearest multiple of 256K. If zero, chunking will be disabled and
// the object will be uploaded in a single request.
//
// ChunkSize will default to a reasonable value. Any custom configuration
// must be done before the first Write call.
ChunkSize int
// ProgressFunc can be used to monitor the progress of a large write.
// operation. If ProgressFunc is not nil and writing requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload),
// then ProgressFunc will be invoked after each call with the number of bytes of
// content copied so far.
//
// ProgressFunc should return quickly without blocking.
ProgressFunc func(int64)
ctx context.Context
o *ObjectHandle
opened bool
pw *io.PipeWriter
donec chan struct{} // closed after err and obj are set.
err error
obj *ObjectAttrs
}
func (w *Writer) open() error {
attrs := w.ObjectAttrs
// Check the developer didn't change the object Name (this is unfortunate, but
// we don't want to store an object under the wrong name).
if attrs.Name != w.o.object {
return fmt.Errorf("storage: Writer.Name %q does not match object name %q", attrs.Name, w.o.object)
}
if !utf8.ValidString(attrs.Name) {
return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name)
}
pr, pw := io.Pipe()
w.pw = pw
w.opened = true
if w.ChunkSize < 0 {
return errors.New("storage: Writer.ChunkSize must non-negative")
}
mediaOpts := []googleapi.MediaOption{
googleapi.ChunkSize(w.ChunkSize),
}
if c := attrs.ContentType; c != "" {
mediaOpts = append(mediaOpts, googleapi.ContentType(c))
}
go func() {
defer close(w.donec)
rawObj := attrs.toRawObject(w.o.bucket)
if w.SendCRC32C {
rawObj.Crc32c = encodeUint32(attrs.CRC32C)
}
if w.MD5 != nil {
rawObj.Md5Hash = base64.StdEncoding.EncodeToString(w.MD5)
}
call := w.o.c.raw.Objects.Insert(w.o.bucket, rawObj).
Media(pr, mediaOpts...).
Projection("full").
Context(w.ctx)
if w.ProgressFunc != nil {
call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) })
}
if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil {
w.err = err
pr.CloseWithError(w.err)
return
}
var resp *raw.Object
err := applyConds("NewWriter", w.o.gen, w.o.conds, call)
if err == nil {
if w.o.userProject != "" {
call.UserProject(w.o.userProject)
}
setClientHeader(call.Header())
// We will only retry here if the initial POST, which obtains a URI for
// the resumable upload, fails with a retryable error. The upload itself
// has its own retry logic.
err = runWithRetry(w.ctx, func() error {
var err2 error
resp, err2 = call.Do()
return err2
})
}
if err != nil {
w.err = err
pr.CloseWithError(w.err)
return
}
w.obj = newObject(resp)
}()
return nil
}
// Write appends to w. It implements the io.Writer interface.
//
// Since writes happen asynchronously, Write may return a nil
// error even though the write failed (or will fail). Always
// use the error returned from Writer.Close to determine if
// the upload was successful.
func (w *Writer) Write(p []byte) (n int, err error) {
if w.err != nil {
return 0, w.err
}
if !w.opened {
if err := w.open(); err != nil {
return 0, err
}
}
return w.pw.Write(p)
}
// Close completes the write operation and flushes any buffered data.
// If Close doesn't return an error, metadata about the written object
// can be retrieved by calling Attrs.
func (w *Writer) Close() error {
if !w.opened {
if err := w.open(); err != nil {
return err
}
}
if err := w.pw.Close(); err != nil {
return err
}
<-w.donec
return w.err
}
// CloseWithError aborts the write operation with the provided error.
// CloseWithError always returns nil.
func (w *Writer) CloseWithError(err error) error {
if !w.opened {
return nil
}
return w.pw.CloseWithError(err)
}
// Attrs returns metadata about a successfully-written object.
// It's only valid to call it after Close returns nil.
func (w *Writer) Attrs() *ObjectAttrs {
return w.obj
}