mirror of
https://github.com/willnorris/imageproxy.git
synced 2026-04-26 21:36:24 +02:00
vendor: add github.com/diegomarangoni/gcscache
also include all transitive dependencies of gcscache, which seems to bring in the entire world of Google Cloud Platform.
This commit is contained in:
parent
fcb6465a29
commit
0fe3656939
251 changed files with 90555 additions and 0 deletions
252
vendor/cloud.google.com/go/storage/acl.go
generated
vendored
Normal file
252
vendor/cloud.google.com/go/storage/acl.go
generated
vendored
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/googleapi"
|
||||
raw "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
// ACLRole is the level of access to grant.
|
||||
type ACLRole string
|
||||
|
||||
const (
|
||||
RoleOwner ACLRole = "OWNER"
|
||||
RoleReader ACLRole = "READER"
|
||||
RoleWriter ACLRole = "WRITER"
|
||||
)
|
||||
|
||||
// ACLEntity refers to a user or group.
|
||||
// They are sometimes referred to as grantees.
|
||||
//
|
||||
// It could be in the form of:
|
||||
// "user-<userId>", "user-<email>", "group-<groupId>", "group-<email>",
|
||||
// "domain-<domain>" and "project-team-<projectId>".
|
||||
//
|
||||
// Or one of the predefined constants: AllUsers, AllAuthenticatedUsers.
|
||||
type ACLEntity string
|
||||
|
||||
const (
|
||||
AllUsers ACLEntity = "allUsers"
|
||||
AllAuthenticatedUsers ACLEntity = "allAuthenticatedUsers"
|
||||
)
|
||||
|
||||
// ACLRule represents a grant for a role to an entity (user, group or team) for a Google Cloud Storage object or bucket.
|
||||
type ACLRule struct {
|
||||
Entity ACLEntity
|
||||
Role ACLRole
|
||||
}
|
||||
|
||||
// ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object.
|
||||
type ACLHandle struct {
|
||||
c *Client
|
||||
bucket string
|
||||
object string
|
||||
isDefault bool
|
||||
userProject string // for requester-pays buckets
|
||||
}
|
||||
|
||||
// Delete permanently deletes the ACL entry for the given entity.
|
||||
func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error {
|
||||
if a.object != "" {
|
||||
return a.objectDelete(ctx, entity)
|
||||
}
|
||||
if a.isDefault {
|
||||
return a.bucketDefaultDelete(ctx, entity)
|
||||
}
|
||||
return a.bucketDelete(ctx, entity)
|
||||
}
|
||||
|
||||
// Set sets the permission level for the given entity.
|
||||
func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error {
|
||||
if a.object != "" {
|
||||
return a.objectSet(ctx, entity, role, false)
|
||||
}
|
||||
if a.isDefault {
|
||||
return a.objectSet(ctx, entity, role, true)
|
||||
}
|
||||
return a.bucketSet(ctx, entity, role)
|
||||
}
|
||||
|
||||
// List retrieves ACL entries.
|
||||
func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) {
|
||||
if a.object != "" {
|
||||
return a.objectList(ctx)
|
||||
}
|
||||
if a.isDefault {
|
||||
return a.bucketDefaultList(ctx)
|
||||
}
|
||||
return a.bucketList(ctx)
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.ObjectAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
|
||||
a.configureCall(req, ctx)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err)
|
||||
}
|
||||
return toACLRules(acls.Items), nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
|
||||
err := runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
|
||||
a.configureCall(req, ctx)
|
||||
return req.Do()
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.BucketAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.BucketAccessControls.List(a.bucket)
|
||||
a.configureCall(req, ctx)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err)
|
||||
}
|
||||
r := make([]ACLRule, len(acls.Items))
|
||||
for i, v := range acls.Items {
|
||||
r[i].Entity = ACLEntity(v.Entity)
|
||||
r[i].Role = ACLRole(v.Role)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
|
||||
acl := &raw.BucketAccessControl{
|
||||
Bucket: a.bucket,
|
||||
Entity: string(entity),
|
||||
Role: string(role),
|
||||
}
|
||||
err := runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
|
||||
a.configureCall(req, ctx)
|
||||
_, err := req.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
|
||||
err := runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
|
||||
a.configureCall(req, ctx)
|
||||
return req.Do()
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
|
||||
var acls *raw.ObjectAccessControls
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
|
||||
a.configureCall(req, ctx)
|
||||
acls, err = req.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err)
|
||||
}
|
||||
return toACLRules(acls.Items), nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error {
|
||||
type setRequest interface {
|
||||
Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error)
|
||||
Header() http.Header
|
||||
}
|
||||
|
||||
acl := &raw.ObjectAccessControl{
|
||||
Bucket: a.bucket,
|
||||
Entity: string(entity),
|
||||
Role: string(role),
|
||||
}
|
||||
var req setRequest
|
||||
if isBucketDefault {
|
||||
req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl)
|
||||
} else {
|
||||
req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
|
||||
}
|
||||
a.configureCall(req, ctx)
|
||||
err := runWithRetry(ctx, func() error {
|
||||
_, err := req.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
if isBucketDefault {
|
||||
return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
|
||||
} else {
|
||||
return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
|
||||
err := runWithRetry(ctx, func() error {
|
||||
req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
|
||||
a.configureCall(req, ctx)
|
||||
return req.Do()
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ACLHandle) configureCall(call interface {
|
||||
Header() http.Header
|
||||
}, ctx context.Context) {
|
||||
vc := reflect.ValueOf(call)
|
||||
vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)})
|
||||
if a.userProject != "" {
|
||||
vc.MethodByName("UserProject").Call([]reflect.Value{reflect.ValueOf(a.userProject)})
|
||||
}
|
||||
setClientHeader(call.Header())
|
||||
}
|
||||
|
||||
func toACLRules(items []*raw.ObjectAccessControl) []ACLRule {
|
||||
r := make([]ACLRule, 0, len(items))
|
||||
for _, item := range items {
|
||||
r = append(r, ACLRule{Entity: ACLEntity(item.Entity), Role: ACLRole(item.Role)})
|
||||
}
|
||||
return r
|
||||
}
|
||||
758
vendor/cloud.google.com/go/storage/bucket.go
generated
vendored
Normal file
758
vendor/cloud.google.com/go/storage/bucket.go
generated
vendored
Normal file
|
|
@ -0,0 +1,758 @@
|
|||
// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/internal/optional"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/iterator"
|
||||
raw "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
// BucketHandle provides operations on a Google Cloud Storage bucket.
|
||||
// Use Client.Bucket to get a handle.
|
||||
type BucketHandle struct {
|
||||
c *Client
|
||||
name string
|
||||
acl ACLHandle
|
||||
defaultObjectACL ACLHandle
|
||||
conds *BucketConditions
|
||||
userProject string // project for requester-pays buckets
|
||||
}
|
||||
|
||||
// Bucket returns a BucketHandle, which provides operations on the named bucket.
|
||||
// This call does not perform any network operations.
|
||||
//
|
||||
// The supplied name must contain only lowercase letters, numbers, dashes,
|
||||
// underscores, and dots. The full specification for valid bucket names can be
|
||||
// found at:
|
||||
// https://cloud.google.com/storage/docs/bucket-naming
|
||||
func (c *Client) Bucket(name string) *BucketHandle {
|
||||
return &BucketHandle{
|
||||
c: c,
|
||||
name: name,
|
||||
acl: ACLHandle{
|
||||
c: c,
|
||||
bucket: name,
|
||||
},
|
||||
defaultObjectACL: ACLHandle{
|
||||
c: c,
|
||||
bucket: name,
|
||||
isDefault: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Create creates the Bucket in the project.
|
||||
// If attrs is nil the API defaults will be used.
|
||||
func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error {
|
||||
var bkt *raw.Bucket
|
||||
if attrs != nil {
|
||||
bkt = attrs.toRawBucket()
|
||||
} else {
|
||||
bkt = &raw.Bucket{}
|
||||
}
|
||||
bkt.Name = b.name
|
||||
// If there is lifecycle information but no location, explicitly set
|
||||
// the location. This is a GCS quirk/bug.
|
||||
if bkt.Location == "" && bkt.Lifecycle != nil {
|
||||
bkt.Location = "US"
|
||||
}
|
||||
req := b.c.raw.Buckets.Insert(projectID, bkt)
|
||||
setClientHeader(req.Header())
|
||||
return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err })
|
||||
}
|
||||
|
||||
// Delete deletes the Bucket.
|
||||
func (b *BucketHandle) Delete(ctx context.Context) error {
|
||||
req, err := b.newDeleteCall()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
|
||||
}
|
||||
|
||||
func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
|
||||
req := b.c.raw.Buckets.Delete(b.name)
|
||||
setClientHeader(req.Header())
|
||||
if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.userProject != "" {
|
||||
req.UserProject(b.userProject)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// ACL returns an ACLHandle, which provides access to the bucket's access control list.
|
||||
// This controls who can list, create or overwrite the objects in a bucket.
|
||||
// This call does not perform any network operations.
|
||||
func (b *BucketHandle) ACL() *ACLHandle {
|
||||
return &b.acl
|
||||
}
|
||||
|
||||
// DefaultObjectACL returns an ACLHandle, which provides access to the bucket's default object ACLs.
|
||||
// These ACLs are applied to newly created objects in this bucket that do not have a defined ACL.
|
||||
// This call does not perform any network operations.
|
||||
func (b *BucketHandle) DefaultObjectACL() *ACLHandle {
|
||||
return &b.defaultObjectACL
|
||||
}
|
||||
|
||||
// Object returns an ObjectHandle, which provides operations on the named object.
|
||||
// This call does not perform any network operations.
|
||||
//
|
||||
// name must consist entirely of valid UTF-8-encoded runes. The full specification
|
||||
// for valid object names can be found at:
|
||||
// https://cloud.google.com/storage/docs/bucket-naming
|
||||
func (b *BucketHandle) Object(name string) *ObjectHandle {
|
||||
return &ObjectHandle{
|
||||
c: b.c,
|
||||
bucket: b.name,
|
||||
object: name,
|
||||
acl: ACLHandle{
|
||||
c: b.c,
|
||||
bucket: b.name,
|
||||
object: name,
|
||||
userProject: b.userProject,
|
||||
},
|
||||
gen: -1,
|
||||
userProject: b.userProject,
|
||||
}
|
||||
}
|
||||
|
||||
// Attrs returns the metadata for the bucket.
|
||||
func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) {
|
||||
req, err := b.newGetCall()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp *raw.Bucket
|
||||
err = runWithRetry(ctx, func() error {
|
||||
resp, err = req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
|
||||
return nil, ErrBucketNotExist
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newBucket(resp), nil
|
||||
}
|
||||
|
||||
func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) {
|
||||
req := b.c.raw.Buckets.Get(b.name).Projection("full")
|
||||
setClientHeader(req.Header())
|
||||
if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.userProject != "" {
|
||||
req.UserProject(b.userProject)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) {
|
||||
req, err := b.newPatchCall(&uattrs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// TODO(jba): retry iff metagen is set?
|
||||
rb, err := req.Context(ctx).Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newBucket(rb), nil
|
||||
}
|
||||
|
||||
func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
|
||||
rb := uattrs.toRawBucket()
|
||||
req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full")
|
||||
setClientHeader(req.Header())
|
||||
if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.userProject != "" {
|
||||
req.UserProject(b.userProject)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
|
||||
type BucketAttrs struct {
|
||||
// Name is the name of the bucket.
|
||||
Name string
|
||||
|
||||
// ACL is the list of access control rules on the bucket.
|
||||
ACL []ACLRule
|
||||
|
||||
// DefaultObjectACL is the list of access controls to
|
||||
// apply to new objects when no object ACL is provided.
|
||||
DefaultObjectACL []ACLRule
|
||||
|
||||
// Location is the location of the bucket. It defaults to "US".
|
||||
Location string
|
||||
|
||||
// MetaGeneration is the metadata generation of the bucket.
|
||||
MetaGeneration int64
|
||||
|
||||
// StorageClass is the default storage class of the bucket. This defines
|
||||
// how objects in the bucket are stored and determines the SLA
|
||||
// and the cost of storage. Typical values are "MULTI_REGIONAL",
|
||||
// "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" and
|
||||
// "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD", which
|
||||
// is equivalent to "MULTI_REGIONAL" or "REGIONAL" depending on
|
||||
// the bucket's location settings.
|
||||
StorageClass string
|
||||
|
||||
// Created is the creation time of the bucket.
|
||||
Created time.Time
|
||||
|
||||
// VersioningEnabled reports whether this bucket has versioning enabled.
|
||||
// This field is read-only.
|
||||
VersioningEnabled bool
|
||||
|
||||
// Labels are the bucket's labels.
|
||||
Labels map[string]string
|
||||
|
||||
// RequesterPays reports whether the bucket is a Requester Pays bucket.
|
||||
RequesterPays bool
|
||||
// Lifecycle is the lifecycle configuration for objects in the bucket.
|
||||
Lifecycle Lifecycle
|
||||
}
|
||||
|
||||
// Lifecycle is the lifecycle configuration for objects in the bucket.
|
||||
type Lifecycle struct {
|
||||
Rules []LifecycleRule
|
||||
}
|
||||
|
||||
const (
|
||||
// RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule.
|
||||
rfc3339Date = "2006-01-02"
|
||||
|
||||
// DeleteAction is a lifecycle action that deletes a live and/or archived
|
||||
// objects. Takes precendence over SetStorageClass actions.
|
||||
DeleteAction = "Delete"
|
||||
|
||||
// SetStorageClassAction changes the storage class of live and/or archived
|
||||
// objects.
|
||||
SetStorageClassAction = "SetStorageClass"
|
||||
)
|
||||
|
||||
// LifecycleRule is a lifecycle configuration rule.
|
||||
//
|
||||
// When all the configured conditions are met by an object in the bucket, the
|
||||
// configured action will automatically be taken on that object.
|
||||
type LifecycleRule struct {
|
||||
// Action is the action to take when all of the associated conditions are
|
||||
// met.
|
||||
Action LifecycleAction
|
||||
|
||||
// Condition is the set of conditions that must be met for the associated
|
||||
// action to be taken.
|
||||
Condition LifecycleCondition
|
||||
}
|
||||
|
||||
// LifecycleAction is a lifecycle configuration action.
|
||||
type LifecycleAction struct {
|
||||
// Type is the type of action to take on matching objects.
|
||||
//
|
||||
// Acceptable values are "Delete" to delete matching objects and
|
||||
// "SetStorageClass" to set the storage class defined in StorageClass on
|
||||
// matching objects.
|
||||
Type string
|
||||
|
||||
// StorageClass is the storage class to set on matching objects if the Action
|
||||
// is "SetStorageClass".
|
||||
StorageClass string
|
||||
}
|
||||
|
||||
// Liveness specifies whether the object is live or not.
|
||||
type Liveness int
|
||||
|
||||
const (
|
||||
// LiveAndArchived includes both live and archived objects.
|
||||
LiveAndArchived Liveness = iota
|
||||
// Live specifies that the object is still live.
|
||||
Live
|
||||
// Archived specifies that the object is archived.
|
||||
Archived
|
||||
)
|
||||
|
||||
// LifecycleCondition is a set of conditions used to match objects and take an
|
||||
// action automatically.
|
||||
//
|
||||
// All configured conditions must be met for the associated action to be taken.
|
||||
type LifecycleCondition struct {
|
||||
// AgeInDays is the age of the object in days.
|
||||
AgeInDays int64
|
||||
|
||||
// CreatedBefore is the time the object was created.
|
||||
//
|
||||
// This condition is satisfied when an object is created before midnight of
|
||||
// the specified date in UTC.
|
||||
CreatedBefore time.Time
|
||||
|
||||
// Liveness specifies the object's liveness. Relevant only for versioned objects
|
||||
Liveness Liveness
|
||||
|
||||
// MatchesStorageClasses is the condition matching the object's storage
|
||||
// class.
|
||||
//
|
||||
// Values include "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE",
|
||||
// "STANDARD", and "DURABLE_REDUCED_AVAILABILITY".
|
||||
MatchesStorageClasses []string
|
||||
|
||||
// NumNewerVersions is the condition matching objects with a number of newer versions.
|
||||
//
|
||||
// If the value is N, this condition is satisfied when there are at least N
|
||||
// versions (including the live version) newer than this version of the
|
||||
// object.
|
||||
NumNewerVersions int64
|
||||
}
|
||||
|
||||
func newBucket(b *raw.Bucket) *BucketAttrs {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
bucket := &BucketAttrs{
|
||||
Name: b.Name,
|
||||
Location: b.Location,
|
||||
MetaGeneration: b.Metageneration,
|
||||
StorageClass: b.StorageClass,
|
||||
Created: convertTime(b.TimeCreated),
|
||||
VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled,
|
||||
Labels: b.Labels,
|
||||
RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
|
||||
Lifecycle: toLifecycle(b.Lifecycle),
|
||||
}
|
||||
acl := make([]ACLRule, len(b.Acl))
|
||||
for i, rule := range b.Acl {
|
||||
acl[i] = ACLRule{
|
||||
Entity: ACLEntity(rule.Entity),
|
||||
Role: ACLRole(rule.Role),
|
||||
}
|
||||
}
|
||||
bucket.ACL = acl
|
||||
objACL := make([]ACLRule, len(b.DefaultObjectAcl))
|
||||
for i, rule := range b.DefaultObjectAcl {
|
||||
objACL[i] = ACLRule{
|
||||
Entity: ACLEntity(rule.Entity),
|
||||
Role: ACLRole(rule.Role),
|
||||
}
|
||||
}
|
||||
bucket.DefaultObjectACL = objACL
|
||||
return bucket
|
||||
}
|
||||
|
||||
// toRawBucket copies the editable attribute from b to the raw library's Bucket type.
|
||||
func (b *BucketAttrs) toRawBucket() *raw.Bucket {
|
||||
var acl []*raw.BucketAccessControl
|
||||
if len(b.ACL) > 0 {
|
||||
acl = make([]*raw.BucketAccessControl, len(b.ACL))
|
||||
for i, rule := range b.ACL {
|
||||
acl[i] = &raw.BucketAccessControl{
|
||||
Entity: string(rule.Entity),
|
||||
Role: string(rule.Role),
|
||||
}
|
||||
}
|
||||
}
|
||||
dACL := toRawObjectACL(b.DefaultObjectACL)
|
||||
// Copy label map.
|
||||
var labels map[string]string
|
||||
if len(b.Labels) > 0 {
|
||||
labels = make(map[string]string, len(b.Labels))
|
||||
for k, v := range b.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
}
|
||||
// Ignore VersioningEnabled if it is false. This is OK because
|
||||
// we only call this method when creating a bucket, and by default
|
||||
// new buckets have versioning off.
|
||||
var v *raw.BucketVersioning
|
||||
if b.VersioningEnabled {
|
||||
v = &raw.BucketVersioning{Enabled: true}
|
||||
}
|
||||
var bb *raw.BucketBilling
|
||||
if b.RequesterPays {
|
||||
bb = &raw.BucketBilling{RequesterPays: true}
|
||||
}
|
||||
return &raw.Bucket{
|
||||
Name: b.Name,
|
||||
DefaultObjectAcl: dACL,
|
||||
Location: b.Location,
|
||||
StorageClass: b.StorageClass,
|
||||
Acl: acl,
|
||||
Versioning: v,
|
||||
Labels: labels,
|
||||
Billing: bb,
|
||||
Lifecycle: toRawLifecycle(b.Lifecycle),
|
||||
}
|
||||
}
|
||||
|
||||
type BucketAttrsToUpdate struct {
|
||||
// VersioningEnabled, if set, updates whether the bucket uses versioning.
|
||||
VersioningEnabled optional.Bool
|
||||
|
||||
// RequesterPays, if set, updates whether the bucket is a Requester Pays bucket.
|
||||
RequesterPays optional.Bool
|
||||
|
||||
setLabels map[string]string
|
||||
deleteLabels map[string]bool
|
||||
}
|
||||
|
||||
// SetLabel causes a label to be added or modified when ua is used
|
||||
// in a call to Bucket.Update.
|
||||
func (ua *BucketAttrsToUpdate) SetLabel(name, value string) {
|
||||
if ua.setLabels == nil {
|
||||
ua.setLabels = map[string]string{}
|
||||
}
|
||||
ua.setLabels[name] = value
|
||||
}
|
||||
|
||||
// DeleteLabel causes a label to be deleted when ua is used in a
|
||||
// call to Bucket.Update.
|
||||
func (ua *BucketAttrsToUpdate) DeleteLabel(name string) {
|
||||
if ua.deleteLabels == nil {
|
||||
ua.deleteLabels = map[string]bool{}
|
||||
}
|
||||
ua.deleteLabels[name] = true
|
||||
}
|
||||
|
||||
func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
|
||||
rb := &raw.Bucket{}
|
||||
if ua.VersioningEnabled != nil {
|
||||
rb.Versioning = &raw.BucketVersioning{
|
||||
Enabled: optional.ToBool(ua.VersioningEnabled),
|
||||
ForceSendFields: []string{"Enabled"},
|
||||
}
|
||||
}
|
||||
if ua.RequesterPays != nil {
|
||||
rb.Billing = &raw.BucketBilling{
|
||||
RequesterPays: optional.ToBool(ua.RequesterPays),
|
||||
ForceSendFields: []string{"RequesterPays"},
|
||||
}
|
||||
}
|
||||
if ua.setLabels != nil || ua.deleteLabels != nil {
|
||||
rb.Labels = map[string]string{}
|
||||
for k, v := range ua.setLabels {
|
||||
rb.Labels[k] = v
|
||||
}
|
||||
if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 {
|
||||
rb.ForceSendFields = append(rb.ForceSendFields, "Labels")
|
||||
}
|
||||
for l := range ua.deleteLabels {
|
||||
rb.NullFields = append(rb.NullFields, "Labels."+l)
|
||||
}
|
||||
}
|
||||
return rb
|
||||
}
|
||||
|
||||
// If returns a new BucketHandle that applies a set of preconditions.
|
||||
// Preconditions already set on the BucketHandle are ignored.
|
||||
// Operations on the new handle will only occur if the preconditions are
|
||||
// satisfied. The only valid preconditions for buckets are MetagenerationMatch
|
||||
// and MetagenerationNotMatch.
|
||||
func (b *BucketHandle) If(conds BucketConditions) *BucketHandle {
|
||||
b2 := *b
|
||||
b2.conds = &conds
|
||||
return &b2
|
||||
}
|
||||
|
||||
// BucketConditions constrain bucket methods to act on specific metagenerations.
|
||||
//
|
||||
// The zero value is an empty set of constraints.
|
||||
type BucketConditions struct {
|
||||
// MetagenerationMatch specifies that the bucket must have the given
|
||||
// metageneration for the operation to occur.
|
||||
// If MetagenerationMatch is zero, it has no effect.
|
||||
MetagenerationMatch int64
|
||||
|
||||
// MetagenerationNotMatch specifies that the bucket must not have the given
|
||||
// metageneration for the operation to occur.
|
||||
// If MetagenerationNotMatch is zero, it has no effect.
|
||||
MetagenerationNotMatch int64
|
||||
}
|
||||
|
||||
func (c *BucketConditions) validate(method string) error {
|
||||
if *c == (BucketConditions{}) {
|
||||
return fmt.Errorf("storage: %s: empty conditions", method)
|
||||
}
|
||||
if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 {
|
||||
return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UserProject returns a new BucketHandle that passes the project ID as the user
|
||||
// project for all subsequent calls. A user project is required for all operations
|
||||
// on requester-pays buckets.
|
||||
func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
|
||||
b2 := *b
|
||||
b2.userProject = projectID
|
||||
b2.acl.userProject = projectID
|
||||
b2.defaultObjectACL.userProject = projectID
|
||||
return &b2
|
||||
}
|
||||
|
||||
// applyBucketConds modifies the provided call using the conditions in conds.
|
||||
// call is something that quacks like a *raw.WhateverCall.
|
||||
func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
|
||||
if conds == nil {
|
||||
return nil
|
||||
}
|
||||
if err := conds.validate(method); err != nil {
|
||||
return err
|
||||
}
|
||||
cval := reflect.ValueOf(call)
|
||||
switch {
|
||||
case conds.MetagenerationMatch != 0:
|
||||
if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
|
||||
return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
|
||||
}
|
||||
case conds.MetagenerationNotMatch != 0:
|
||||
if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
|
||||
return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
|
||||
var rl raw.BucketLifecycle
|
||||
if len(l.Rules) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, r := range l.Rules {
|
||||
rr := &raw.BucketLifecycleRule{
|
||||
Action: &raw.BucketLifecycleRuleAction{
|
||||
Type: r.Action.Type,
|
||||
StorageClass: r.Action.StorageClass,
|
||||
},
|
||||
Condition: &raw.BucketLifecycleRuleCondition{
|
||||
Age: r.Condition.AgeInDays,
|
||||
MatchesStorageClass: r.Condition.MatchesStorageClasses,
|
||||
NumNewerVersions: r.Condition.NumNewerVersions,
|
||||
},
|
||||
}
|
||||
|
||||
switch r.Condition.Liveness {
|
||||
case LiveAndArchived:
|
||||
rr.Condition.IsLive = nil
|
||||
case Live:
|
||||
rr.Condition.IsLive = googleapi.Bool(true)
|
||||
case Archived:
|
||||
rr.Condition.IsLive = googleapi.Bool(false)
|
||||
}
|
||||
|
||||
if !r.Condition.CreatedBefore.IsZero() {
|
||||
rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date)
|
||||
}
|
||||
rl.Rule = append(rl.Rule, rr)
|
||||
}
|
||||
return &rl
|
||||
}
|
||||
|
||||
func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
|
||||
var l Lifecycle
|
||||
if rl == nil {
|
||||
return l
|
||||
}
|
||||
for _, rr := range rl.Rule {
|
||||
r := LifecycleRule{
|
||||
Action: LifecycleAction{
|
||||
Type: rr.Action.Type,
|
||||
StorageClass: rr.Action.StorageClass,
|
||||
},
|
||||
Condition: LifecycleCondition{
|
||||
AgeInDays: rr.Condition.Age,
|
||||
MatchesStorageClasses: rr.Condition.MatchesStorageClass,
|
||||
NumNewerVersions: rr.Condition.NumNewerVersions,
|
||||
},
|
||||
}
|
||||
|
||||
switch {
|
||||
case rr.Condition.IsLive == nil:
|
||||
r.Condition.Liveness = LiveAndArchived
|
||||
case *rr.Condition.IsLive == true:
|
||||
r.Condition.Liveness = Live
|
||||
case *rr.Condition.IsLive == false:
|
||||
r.Condition.Liveness = Archived
|
||||
}
|
||||
|
||||
if rr.Condition.CreatedBefore != "" {
|
||||
r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
|
||||
}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Objects returns an iterator over the objects in the bucket that match the Query q.
|
||||
// If q is nil, no filtering is done.
|
||||
func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
|
||||
it := &ObjectIterator{
|
||||
ctx: ctx,
|
||||
bucket: b,
|
||||
}
|
||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
||||
it.fetch,
|
||||
func() int { return len(it.items) },
|
||||
func() interface{} { b := it.items; it.items = nil; return b })
|
||||
if q != nil {
|
||||
it.query = *q
|
||||
}
|
||||
return it
|
||||
}
|
||||
|
||||
// An ObjectIterator is an iterator over ObjectAttrs.
|
||||
type ObjectIterator struct {
|
||||
ctx context.Context
|
||||
bucket *BucketHandle
|
||||
query Query
|
||||
pageInfo *iterator.PageInfo
|
||||
nextFunc func() error
|
||||
items []*ObjectAttrs
|
||||
}
|
||||
|
||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
||||
func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
||||
|
||||
// Next returns the next result. Its second return value is iterator.Done if
|
||||
// there are no more results. Once Next returns iterator.Done, all subsequent
|
||||
// calls will return iterator.Done.
|
||||
//
|
||||
// If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will
|
||||
// have a non-empty Prefix field, and a zero value for all other fields. These
|
||||
// represent prefixes.
|
||||
func (it *ObjectIterator) Next() (*ObjectAttrs, error) {
|
||||
if err := it.nextFunc(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item := it.items[0]
|
||||
it.items = it.items[1:]
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) {
|
||||
req := it.bucket.c.raw.Objects.List(it.bucket.name)
|
||||
setClientHeader(req.Header())
|
||||
req.Projection("full")
|
||||
req.Delimiter(it.query.Delimiter)
|
||||
req.Prefix(it.query.Prefix)
|
||||
req.Versions(it.query.Versions)
|
||||
req.PageToken(pageToken)
|
||||
if it.bucket.userProject != "" {
|
||||
req.UserProject(it.bucket.userProject)
|
||||
}
|
||||
if pageSize > 0 {
|
||||
req.MaxResults(int64(pageSize))
|
||||
}
|
||||
var resp *raw.Objects
|
||||
var err error
|
||||
err = runWithRetry(it.ctx, func() error {
|
||||
resp, err = req.Context(it.ctx).Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound {
|
||||
err = ErrBucketNotExist
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
for _, item := range resp.Items {
|
||||
it.items = append(it.items, newObject(item))
|
||||
}
|
||||
for _, prefix := range resp.Prefixes {
|
||||
it.items = append(it.items, &ObjectAttrs{Prefix: prefix})
|
||||
}
|
||||
return resp.NextPageToken, nil
|
||||
}
|
||||
|
||||
// TODO(jbd): Add storage.buckets.update.
|
||||
|
||||
// Buckets returns an iterator over the buckets in the project. You may
|
||||
// optionally set the iterator's Prefix field to restrict the list to buckets
|
||||
// whose names begin with the prefix. By default, all buckets in the project
|
||||
// are returned.
|
||||
func (c *Client) Buckets(ctx context.Context, projectID string) *BucketIterator {
|
||||
it := &BucketIterator{
|
||||
ctx: ctx,
|
||||
client: c,
|
||||
projectID: projectID,
|
||||
}
|
||||
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
|
||||
it.fetch,
|
||||
func() int { return len(it.buckets) },
|
||||
func() interface{} { b := it.buckets; it.buckets = nil; return b })
|
||||
return it
|
||||
}
|
||||
|
||||
// A BucketIterator is an iterator over BucketAttrs.
|
||||
type BucketIterator struct {
|
||||
// Prefix restricts the iterator to buckets whose names begin with it.
|
||||
Prefix string
|
||||
|
||||
ctx context.Context
|
||||
client *Client
|
||||
projectID string
|
||||
buckets []*BucketAttrs
|
||||
pageInfo *iterator.PageInfo
|
||||
nextFunc func() error
|
||||
}
|
||||
|
||||
// Next returns the next result. Its second return value is iterator.Done if
|
||||
// there are no more results. Once Next returns iterator.Done, all subsequent
|
||||
// calls will return iterator.Done.
|
||||
func (it *BucketIterator) Next() (*BucketAttrs, error) {
|
||||
if err := it.nextFunc(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := it.buckets[0]
|
||||
it.buckets = it.buckets[1:]
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
|
||||
func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
|
||||
|
||||
func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) {
|
||||
req := it.client.raw.Buckets.List(it.projectID)
|
||||
setClientHeader(req.Header())
|
||||
req.Projection("full")
|
||||
req.Prefix(it.Prefix)
|
||||
req.PageToken(pageToken)
|
||||
if pageSize > 0 {
|
||||
req.MaxResults(int64(pageSize))
|
||||
}
|
||||
var resp *raw.Buckets
|
||||
var err error
|
||||
err = runWithRetry(it.ctx, func() error {
|
||||
resp, err = req.Context(it.ctx).Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, item := range resp.Items {
|
||||
it.buckets = append(it.buckets, newBucket(item))
|
||||
}
|
||||
return resp.NextPageToken, nil
|
||||
}
|
||||
201
vendor/cloud.google.com/go/storage/copy.go
generated
vendored
Normal file
201
vendor/cloud.google.com/go/storage/copy.go
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
raw "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
// CopierFrom creates a Copier that can copy src to dst.
|
||||
// You can immediately call Run on the returned Copier, or
|
||||
// you can configure it first.
|
||||
//
|
||||
// For Requester Pays buckets, the user project of dst is billed, unless it is empty,
|
||||
// in which case the user project of src is billed.
|
||||
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
|
||||
return &Copier{dst: dst, src: src}
|
||||
}
|
||||
|
||||
// A Copier copies a source object to a destination.
|
||||
type Copier struct {
|
||||
// ObjectAttrs are optional attributes to set on the destination object.
|
||||
// Any attributes must be initialized before any calls on the Copier. Nil
|
||||
// or zero-valued attributes are ignored.
|
||||
ObjectAttrs
|
||||
|
||||
// RewriteToken can be set before calling Run to resume a copy
|
||||
// operation. After Run returns a non-nil error, RewriteToken will
|
||||
// have been updated to contain the value needed to resume the copy.
|
||||
RewriteToken string
|
||||
|
||||
// ProgressFunc can be used to monitor the progress of a multi-RPC copy
|
||||
// operation. If ProgressFunc is not nil and copying requires multiple
|
||||
// calls to the underlying service (see
|
||||
// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
|
||||
// ProgressFunc will be invoked after each call with the number of bytes of
|
||||
// content copied so far and the total size in bytes of the source object.
|
||||
//
|
||||
// ProgressFunc is intended to make upload progress available to the
|
||||
// application. For example, the implementation of ProgressFunc may update
|
||||
// a progress bar in the application's UI, or log the result of
|
||||
// float64(copiedBytes)/float64(totalBytes).
|
||||
//
|
||||
// ProgressFunc should return quickly without blocking.
|
||||
ProgressFunc func(copiedBytes, totalBytes uint64)
|
||||
|
||||
dst, src *ObjectHandle
|
||||
}
|
||||
|
||||
// Run performs the copy.
|
||||
func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
|
||||
if err := c.src.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := c.dst.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Convert destination attributes to raw form, omitting the bucket.
|
||||
// If the bucket is included but name or content-type aren't, the service
|
||||
// returns a 400 with "Required" as the only message. Omitting the bucket
|
||||
// does not cause any problems.
|
||||
rawObject := c.ObjectAttrs.toRawObject("")
|
||||
for {
|
||||
res, err := c.callRewrite(ctx, rawObject)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.ProgressFunc != nil {
|
||||
c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize))
|
||||
}
|
||||
if res.Done { // Finished successfully.
|
||||
return newObject(res.Resource), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) {
|
||||
call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj)
|
||||
|
||||
call.Context(ctx).Projection("full")
|
||||
if c.RewriteToken != "" {
|
||||
call.RewriteToken(c.RewriteToken)
|
||||
}
|
||||
if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.dst.userProject != "" {
|
||||
call.UserProject(c.dst.userProject)
|
||||
} else if c.src.userProject != "" {
|
||||
call.UserProject(c.src.userProject)
|
||||
}
|
||||
if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := setEncryptionHeaders(call.Header(), c.src.encryptionKey, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res *raw.RewriteResponse
|
||||
var err error
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { res, err = call.Do(); return err })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.RewriteToken = res.RewriteToken
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// ComposerFrom creates a Composer that can compose srcs into dst.
|
||||
// You can immediately call Run on the returned Composer, or you can
|
||||
// configure it first.
|
||||
//
|
||||
// The encryption key for the destination object will be used to decrypt all
|
||||
// source objects and encrypt the destination object. It is an error
|
||||
// to specify an encryption key for any of the source objects.
|
||||
func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
|
||||
return &Composer{dst: dst, srcs: srcs}
|
||||
}
|
||||
|
||||
// A Composer composes source objects into a destination object.
|
||||
//
|
||||
// For Requester Pays buckets, the user project of dst is billed.
|
||||
type Composer struct {
|
||||
// ObjectAttrs are optional attributes to set on the destination object.
|
||||
// Any attributes must be initialized before any calls on the Composer. Nil
|
||||
// or zero-valued attributes are ignored.
|
||||
ObjectAttrs
|
||||
|
||||
dst *ObjectHandle
|
||||
srcs []*ObjectHandle
|
||||
}
|
||||
|
||||
// Run performs the compose operation.
|
||||
func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
|
||||
if err := c.dst.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(c.srcs) == 0 {
|
||||
return nil, errors.New("storage: at least one source object must be specified")
|
||||
}
|
||||
|
||||
req := &raw.ComposeRequest{}
|
||||
// Compose requires a non-empty Destination, so we always set it,
|
||||
// even if the caller-provided ObjectAttrs is the zero value.
|
||||
req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket)
|
||||
for _, src := range c.srcs {
|
||||
if err := src.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if src.bucket != c.dst.bucket {
|
||||
return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket)
|
||||
}
|
||||
if src.encryptionKey != nil {
|
||||
return nil, fmt.Errorf("storage: compose source %s.%s must not have encryption key", src.bucket, src.object)
|
||||
}
|
||||
srcObj := &raw.ComposeRequestSourceObjects{
|
||||
Name: src.object,
|
||||
}
|
||||
if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.SourceObjects = append(req.SourceObjects, srcObj)
|
||||
}
|
||||
|
||||
call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx)
|
||||
if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.dst.userProject != "" {
|
||||
call.UserProject(c.dst.userProject)
|
||||
}
|
||||
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var obj *raw.Object
|
||||
var err error
|
||||
setClientHeader(call.Header())
|
||||
err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newObject(obj), nil
|
||||
}
|
||||
161
vendor/cloud.google.com/go/storage/doc.go
generated
vendored
Normal file
161
vendor/cloud.google.com/go/storage/doc.go
generated
vendored
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/*
|
||||
Package storage provides an easy way to work with Google Cloud Storage.
|
||||
Google Cloud Storage stores data in named objects, which are grouped into buckets.
|
||||
|
||||
More information about Google Cloud Storage is available at
|
||||
https://cloud.google.com/storage/docs.
|
||||
|
||||
All of the methods of this package use exponential backoff to retry calls
|
||||
that fail with certain errors, as described in
|
||||
https://cloud.google.com/storage/docs/exponential-backoff.
|
||||
|
||||
Note: This package is in beta. Some backwards-incompatible changes may occur.
|
||||
|
||||
|
||||
Creating a Client
|
||||
|
||||
To start working with this package, create a client:
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
|
||||
Buckets
|
||||
|
||||
A Google Cloud Storage bucket is a collection of objects. To work with a
|
||||
bucket, make a bucket handle:
|
||||
|
||||
bkt := client.Bucket(bucketName)
|
||||
|
||||
A handle is a reference to a bucket. You can have a handle even if the
|
||||
bucket doesn't exist yet. To create a bucket in Google Cloud Storage,
|
||||
call Create on the handle:
|
||||
|
||||
if err := bkt.Create(ctx, projectID, nil); err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
|
||||
Note that although buckets are associated with projects, bucket names are
|
||||
global across all projects.
|
||||
|
||||
Each bucket has associated metadata, represented in this package by
|
||||
BucketAttrs. The third argument to BucketHandle.Create allows you to set
|
||||
the intial BucketAttrs of a bucket. To retrieve a bucket's attributes, use
|
||||
Attrs:
|
||||
|
||||
attrs, err := bkt.Attrs(ctx)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
fmt.Printf("bucket %s, created at %s, is located in %s with storage class %s\n",
|
||||
attrs.Name, attrs.Created, attrs.Location, attrs.StorageClass)
|
||||
|
||||
Objects
|
||||
|
||||
An object holds arbitrary data as a sequence of bytes, like a file. You
|
||||
refer to objects using a handle, just as with buckets. You can use the
|
||||
standard Go io.Reader and io.Writer interfaces to read and write
|
||||
object data:
|
||||
|
||||
obj := bkt.Object("data")
|
||||
// Write something to obj.
|
||||
// w implements io.Writer.
|
||||
w := obj.NewWriter(ctx)
|
||||
// Write some text to obj. This will overwrite whatever is there.
|
||||
if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
// Close, just like writing a file.
|
||||
if err := w.Close(); err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
|
||||
// Read it back.
|
||||
r, err := obj.NewReader(ctx)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
defer r.Close()
|
||||
if _, err := io.Copy(os.Stdout, r); err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
// Prints "This object contains text."
|
||||
|
||||
Objects also have attributes, which you can fetch with Attrs:
|
||||
|
||||
objAttrs, err := obj.Attrs(ctx)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
fmt.Printf("object %s has size %d and can be read using %s\n",
|
||||
objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)
|
||||
|
||||
ACLs
|
||||
|
||||
Both objects and buckets have ACLs (Access Control Lists). An ACL is a list of
|
||||
ACLRules, each of which specifies the role of a user, group or project. ACLs
|
||||
are suitable for fine-grained control, but you may prefer using IAM to control
|
||||
access at the project level (see
|
||||
https://cloud.google.com/storage/docs/access-control/iam).
|
||||
|
||||
To list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:
|
||||
|
||||
acls, err := obj.ACL().List(ctx)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
for _, rule := range acls {
|
||||
fmt.Printf("%s has role %s\n", rule.Entity, rule.Role)
|
||||
}
|
||||
|
||||
You can also set and delete ACLs.
|
||||
|
||||
Conditions
|
||||
|
||||
Every object has a generation and a metageneration. The generation changes
|
||||
whenever the content changes, and the metageneration changes whenever the
|
||||
metadata changes. Conditions let you check these values before an operation;
|
||||
the operation only executes if the conditions match. You can use conditions to
|
||||
prevent race conditions in read-modify-write operations.
|
||||
|
||||
For example, say you've read an object's metadata into objAttrs. Now
|
||||
you want to write to that object, but only if its contents haven't changed
|
||||
since you read it. Here is how to express that:
|
||||
|
||||
w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)
|
||||
// Proceed with writing as above.
|
||||
|
||||
Signed URLs
|
||||
|
||||
You can obtain a URL that lets anyone read or write an object for a limited time.
|
||||
You don't need to create a client to do this. See the documentation of
|
||||
SignedURL for details.
|
||||
|
||||
url, err := storage.SignedURL(bucketName, "shared-object", opts)
|
||||
if err != nil {
|
||||
// TODO: Handle error.
|
||||
}
|
||||
fmt.Println(url)
|
||||
|
||||
Authentication
|
||||
|
||||
See examples of authorization and authentication at
|
||||
https://godoc.org/cloud.google.com/go#pkg-examples.
|
||||
*/
|
||||
package storage // import "cloud.google.com/go/storage"
|
||||
30
vendor/cloud.google.com/go/storage/go110.go
generated
vendored
Normal file
30
vendor/cloud.google.com/go/storage/go110.go
generated
vendored
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build go1.10
|
||||
|
||||
package storage
|
||||
|
||||
import "google.golang.org/api/googleapi"
|
||||
|
||||
func shouldRetry(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *googleapi.Error:
|
||||
// Retry on 429 and 5xx, according to
|
||||
// https://cloud.google.com/storage/docs/exponential-backoff.
|
||||
return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
26
vendor/cloud.google.com/go/storage/go17.go
generated
vendored
Normal file
26
vendor/cloud.google.com/go/storage/go17.go
generated
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build go1.7
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func withContext(r *http.Request, ctx context.Context) *http.Request {
|
||||
return r.WithContext(ctx)
|
||||
}
|
||||
108
vendor/cloud.google.com/go/storage/iam.go
generated
vendored
Normal file
108
vendor/cloud.google.com/go/storage/iam.go
generated
vendored
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/iam"
|
||||
"golang.org/x/net/context"
|
||||
raw "google.golang.org/api/storage/v1"
|
||||
iampb "google.golang.org/genproto/googleapis/iam/v1"
|
||||
)
|
||||
|
||||
// IAM provides access to IAM access control for the bucket.
|
||||
func (b *BucketHandle) IAM() *iam.Handle {
|
||||
return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name)
|
||||
}
|
||||
|
||||
// iamClient implements the iam.client interface.
|
||||
type iamClient struct {
|
||||
raw *raw.Service
|
||||
}
|
||||
|
||||
func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) {
|
||||
req := c.raw.Buckets.GetIamPolicy(resource)
|
||||
setClientHeader(req.Header())
|
||||
var rp *raw.Policy
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
rp, err = req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return iamFromStoragePolicy(rp), nil
|
||||
}
|
||||
|
||||
func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error {
|
||||
rp := iamToStoragePolicy(p)
|
||||
req := c.raw.Buckets.SetIamPolicy(resource, rp)
|
||||
setClientHeader(req.Header())
|
||||
return runWithRetry(ctx, func() error {
|
||||
_, err := req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
|
||||
req := c.raw.Buckets.TestIamPermissions(resource, perms)
|
||||
setClientHeader(req.Header())
|
||||
var res *raw.TestIamPermissionsResponse
|
||||
var err error
|
||||
err = runWithRetry(ctx, func() error {
|
||||
res, err = req.Context(ctx).Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res.Permissions, nil
|
||||
}
|
||||
|
||||
func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy {
|
||||
return &raw.Policy{
|
||||
Bindings: iamToStorageBindings(ip.Bindings),
|
||||
Etag: string(ip.Etag),
|
||||
}
|
||||
}
|
||||
|
||||
func iamToStorageBindings(ibs []*iampb.Binding) []*raw.PolicyBindings {
|
||||
var rbs []*raw.PolicyBindings
|
||||
for _, ib := range ibs {
|
||||
rbs = append(rbs, &raw.PolicyBindings{
|
||||
Role: ib.Role,
|
||||
Members: ib.Members,
|
||||
})
|
||||
}
|
||||
return rbs
|
||||
}
|
||||
|
||||
func iamFromStoragePolicy(rp *raw.Policy) *iampb.Policy {
|
||||
return &iampb.Policy{
|
||||
Bindings: iamFromStorageBindings(rp.Bindings),
|
||||
Etag: []byte(rp.Etag),
|
||||
}
|
||||
}
|
||||
|
||||
func iamFromStorageBindings(rbs []*raw.PolicyBindings) []*iampb.Binding {
|
||||
var ibs []*iampb.Binding
|
||||
for _, rb := range rbs {
|
||||
ibs = append(ibs, &iampb.Binding{
|
||||
Role: rb.Role,
|
||||
Members: rb.Members,
|
||||
})
|
||||
}
|
||||
return ibs
|
||||
}
|
||||
36
vendor/cloud.google.com/go/storage/invoke.go
generated
vendored
Normal file
36
vendor/cloud.google.com/go/storage/invoke.go
generated
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/internal"
|
||||
gax "github.com/googleapis/gax-go"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// runWithRetry calls the function until it returns nil or a non-retryable error, or
|
||||
// the context is done.
|
||||
func runWithRetry(ctx context.Context, call func() error) error {
|
||||
return internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
|
||||
err = call()
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if shouldRetry(err) {
|
||||
return false, nil
|
||||
}
|
||||
return true, err
|
||||
})
|
||||
}
|
||||
40
vendor/cloud.google.com/go/storage/not_go110.go
generated
vendored
Normal file
40
vendor/cloud.google.com/go/storage/not_go110.go
generated
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !go1.10
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
func shouldRetry(err error) bool {
|
||||
switch e := err.(type) {
|
||||
case *googleapi.Error:
|
||||
// Retry on 429 and 5xx, according to
|
||||
// https://cloud.google.com/storage/docs/exponential-backoff.
|
||||
return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
|
||||
case *url.Error:
|
||||
// Retry on REFUSED_STREAM.
|
||||
// Unfortunately the error type is unexported, so we resort to string
|
||||
// matching.
|
||||
return strings.Contains(e.Error(), "REFUSED_STREAM")
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
26
vendor/cloud.google.com/go/storage/not_go17.go
generated
vendored
Normal file
26
vendor/cloud.google.com/go/storage/not_go17.go
generated
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !go1.7
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func withContext(r *http.Request, _ interface{}) *http.Request {
|
||||
// In Go 1.6 and below, ignore the context.
|
||||
return r
|
||||
}
|
||||
80
vendor/cloud.google.com/go/storage/reader.go
generated
vendored
Normal file
80
vendor/cloud.google.com/go/storage/reader.go
generated
vendored
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
)
|
||||
|
||||
var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// Reader reads a Cloud Storage object.
|
||||
// It implements io.Reader.
|
||||
type Reader struct {
|
||||
body io.ReadCloser
|
||||
remain, size int64
|
||||
contentType string
|
||||
cacheControl string
|
||||
checkCRC bool // should we check the CRC?
|
||||
wantCRC uint32 // the CRC32c value the server sent in the header
|
||||
gotCRC uint32 // running crc
|
||||
}
|
||||
|
||||
// Close closes the Reader. It must be called when done reading.
|
||||
func (r *Reader) Close() error {
|
||||
return r.body.Close()
|
||||
}
|
||||
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.body.Read(p)
|
||||
if r.remain != -1 {
|
||||
r.remain -= int64(n)
|
||||
}
|
||||
if r.checkCRC {
|
||||
r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n])
|
||||
// Check CRC here. It would be natural to check it in Close, but
|
||||
// everybody defers Close on the assumption that it doesn't return
|
||||
// anything worth looking at.
|
||||
if r.remain == 0 && r.gotCRC != r.wantCRC {
|
||||
return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d",
|
||||
r.gotCRC, r.wantCRC)
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Size returns the size of the object in bytes.
|
||||
// The returned value is always the same and is not affected by
|
||||
// calls to Read or Close.
|
||||
func (r *Reader) Size() int64 {
|
||||
return r.size
|
||||
}
|
||||
|
||||
// Remain returns the number of bytes left to read, or -1 if unknown.
|
||||
func (r *Reader) Remain() int64 {
|
||||
return r.remain
|
||||
}
|
||||
|
||||
// ContentType returns the content type of the object.
|
||||
func (r *Reader) ContentType() string {
|
||||
return r.contentType
|
||||
}
|
||||
|
||||
// CacheControl returns the cache control of the object.
|
||||
func (r *Reader) CacheControl() string {
|
||||
return r.cacheControl
|
||||
}
|
||||
1124
vendor/cloud.google.com/go/storage/storage.go
generated
vendored
Normal file
1124
vendor/cloud.google.com/go/storage/storage.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
194
vendor/cloud.google.com/go/storage/writer.go
generated
vendored
Normal file
194
vendor/cloud.google.com/go/storage/writer.go
generated
vendored
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/googleapi"
|
||||
raw "google.golang.org/api/storage/v1"
|
||||
)
|
||||
|
||||
// A Writer writes a Cloud Storage object.
|
||||
type Writer struct {
|
||||
// ObjectAttrs are optional attributes to set on the object. Any attributes
|
||||
// must be initialized before the first Write call. Nil or zero-valued
|
||||
// attributes are ignored.
|
||||
ObjectAttrs
|
||||
|
||||
// SendCRC specifies whether to transmit a CRC32C field. It should be set
|
||||
// to true in addition to setting the Writer's CRC32C field, because zero
|
||||
// is a valid CRC and normally a zero would not be transmitted.
|
||||
// If a CRC32C is sent, and the data written does not match the checksum,
|
||||
// the write will be rejected.
|
||||
SendCRC32C bool
|
||||
|
||||
// ChunkSize controls the maximum number of bytes of the object that the
|
||||
// Writer will attempt to send to the server in a single request. Objects
|
||||
// smaller than the size will be sent in a single request, while larger
|
||||
// objects will be split over multiple requests. The size will be rounded up
|
||||
// to the nearest multiple of 256K. If zero, chunking will be disabled and
|
||||
// the object will be uploaded in a single request.
|
||||
//
|
||||
// ChunkSize will default to a reasonable value. Any custom configuration
|
||||
// must be done before the first Write call.
|
||||
ChunkSize int
|
||||
|
||||
// ProgressFunc can be used to monitor the progress of a large write.
|
||||
// operation. If ProgressFunc is not nil and writing requires multiple
|
||||
// calls to the underlying service (see
|
||||
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload),
|
||||
// then ProgressFunc will be invoked after each call with the number of bytes of
|
||||
// content copied so far.
|
||||
//
|
||||
// ProgressFunc should return quickly without blocking.
|
||||
ProgressFunc func(int64)
|
||||
|
||||
ctx context.Context
|
||||
o *ObjectHandle
|
||||
|
||||
opened bool
|
||||
pw *io.PipeWriter
|
||||
|
||||
donec chan struct{} // closed after err and obj are set.
|
||||
err error
|
||||
obj *ObjectAttrs
|
||||
}
|
||||
|
||||
func (w *Writer) open() error {
|
||||
attrs := w.ObjectAttrs
|
||||
// Check the developer didn't change the object Name (this is unfortunate, but
|
||||
// we don't want to store an object under the wrong name).
|
||||
if attrs.Name != w.o.object {
|
||||
return fmt.Errorf("storage: Writer.Name %q does not match object name %q", attrs.Name, w.o.object)
|
||||
}
|
||||
if !utf8.ValidString(attrs.Name) {
|
||||
return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name)
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
w.pw = pw
|
||||
w.opened = true
|
||||
|
||||
if w.ChunkSize < 0 {
|
||||
return errors.New("storage: Writer.ChunkSize must non-negative")
|
||||
}
|
||||
mediaOpts := []googleapi.MediaOption{
|
||||
googleapi.ChunkSize(w.ChunkSize),
|
||||
}
|
||||
if c := attrs.ContentType; c != "" {
|
||||
mediaOpts = append(mediaOpts, googleapi.ContentType(c))
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(w.donec)
|
||||
|
||||
rawObj := attrs.toRawObject(w.o.bucket)
|
||||
if w.SendCRC32C {
|
||||
rawObj.Crc32c = encodeUint32(attrs.CRC32C)
|
||||
}
|
||||
if w.MD5 != nil {
|
||||
rawObj.Md5Hash = base64.StdEncoding.EncodeToString(w.MD5)
|
||||
}
|
||||
call := w.o.c.raw.Objects.Insert(w.o.bucket, rawObj).
|
||||
Media(pr, mediaOpts...).
|
||||
Projection("full").
|
||||
Context(w.ctx)
|
||||
if w.ProgressFunc != nil {
|
||||
call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) })
|
||||
}
|
||||
if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil {
|
||||
w.err = err
|
||||
pr.CloseWithError(w.err)
|
||||
return
|
||||
}
|
||||
var resp *raw.Object
|
||||
err := applyConds("NewWriter", w.o.gen, w.o.conds, call)
|
||||
if err == nil {
|
||||
if w.o.userProject != "" {
|
||||
call.UserProject(w.o.userProject)
|
||||
}
|
||||
setClientHeader(call.Header())
|
||||
// We will only retry here if the initial POST, which obtains a URI for
|
||||
// the resumable upload, fails with a retryable error. The upload itself
|
||||
// has its own retry logic.
|
||||
err = runWithRetry(w.ctx, func() error {
|
||||
var err2 error
|
||||
resp, err2 = call.Do()
|
||||
return err2
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
w.err = err
|
||||
pr.CloseWithError(w.err)
|
||||
return
|
||||
}
|
||||
w.obj = newObject(resp)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write appends to w. It implements the io.Writer interface.
|
||||
//
|
||||
// Since writes happen asynchronously, Write may return a nil
|
||||
// error even though the write failed (or will fail). Always
|
||||
// use the error returned from Writer.Close to determine if
|
||||
// the upload was successful.
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
if !w.opened {
|
||||
if err := w.open(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return w.pw.Write(p)
|
||||
}
|
||||
|
||||
// Close completes the write operation and flushes any buffered data.
|
||||
// If Close doesn't return an error, metadata about the written object
|
||||
// can be retrieved by calling Attrs.
|
||||
func (w *Writer) Close() error {
|
||||
if !w.opened {
|
||||
if err := w.open(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := w.pw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
<-w.donec
|
||||
return w.err
|
||||
}
|
||||
|
||||
// CloseWithError aborts the write operation with the provided error.
|
||||
// CloseWithError always returns nil.
|
||||
func (w *Writer) CloseWithError(err error) error {
|
||||
if !w.opened {
|
||||
return nil
|
||||
}
|
||||
return w.pw.CloseWithError(err)
|
||||
}
|
||||
|
||||
// Attrs returns metadata about a successfully-written object.
|
||||
// It's only valid to call it after Close returns nil.
|
||||
func (w *Writer) Attrs() *ObjectAttrs {
|
||||
return w.obj
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue