vendor: add packages needed for azure support

This commit is contained in:
Will Norris 2017-09-06 00:04:23 +00:00
parent f1ef8af9ec
commit 76c8498abe
84 changed files with 13523 additions and 0 deletions

View file

@ -0,0 +1,72 @@
package storage
import (
"bytes"
"fmt"
"net/http"
"net/url"
"time"
)
// PutAppendBlob initializes an empty append blob with specified name. An
// append blob must be created using this method before appending blocks.
//
// See CreateBlockBlobFromReader for more info on creating blobs.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
func (b *Blob) PutAppendBlob(options *PutBlobOptions) error {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeAppend)
headers = mergeHeaders(headers, headersFromStruct(b.Properties))
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// AppendBlockOptions includes the options for an append block operation
type AppendBlockOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
MaxSize *uint `header:"x-ms-blob-condition-maxsize"`
AppendPosition *uint `header:"x-ms-blob-condition-appendpos"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// AppendBlock appends a block to an append blob.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Append-Block
func (b *Blob) AppendBlock(chunk []byte, options *AppendBlockOptions) error {
params := url.Values{"comp": {"appendblock"}}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeAppend)
headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, bytes.NewReader(chunk), b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}

View file

@ -0,0 +1,230 @@
// Package storage provides clients for Microsoft Azure Storage Services.
package storage
import (
"bytes"
"fmt"
"net/url"
"sort"
"strings"
)
// See: https://docs.microsoft.com/rest/api/storageservices/fileservices/authentication-for-the-azure-storage-services
type authentication string
const (
sharedKey authentication = "sharedKey"
sharedKeyForTable authentication = "sharedKeyTable"
sharedKeyLite authentication = "sharedKeyLite"
sharedKeyLiteForTable authentication = "sharedKeyLiteTable"
// headers
headerAcceptCharset = "Accept-Charset"
headerAuthorization = "Authorization"
headerContentLength = "Content-Length"
headerDate = "Date"
headerXmsDate = "x-ms-date"
headerXmsVersion = "x-ms-version"
headerContentEncoding = "Content-Encoding"
headerContentLanguage = "Content-Language"
headerContentType = "Content-Type"
headerContentMD5 = "Content-MD5"
headerIfModifiedSince = "If-Modified-Since"
headerIfMatch = "If-Match"
headerIfNoneMatch = "If-None-Match"
headerIfUnmodifiedSince = "If-Unmodified-Since"
headerRange = "Range"
headerDataServiceVersion = "DataServiceVersion"
headerMaxDataServiceVersion = "MaxDataServiceVersion"
headerContentTransferEncoding = "Content-Transfer-Encoding"
)
func (c *Client) addAuthorizationHeader(verb, url string, headers map[string]string, auth authentication) (map[string]string, error) {
authHeader, err := c.getSharedKey(verb, url, headers, auth)
if err != nil {
return nil, err
}
headers[headerAuthorization] = authHeader
return headers, nil
}
func (c *Client) getSharedKey(verb, url string, headers map[string]string, auth authentication) (string, error) {
canRes, err := c.buildCanonicalizedResource(url, auth, false)
if err != nil {
return "", err
}
canString, err := buildCanonicalizedString(verb, headers, canRes, auth)
if err != nil {
return "", err
}
return c.createAuthorizationHeader(canString, auth), nil
}
func (c *Client) buildCanonicalizedResource(uri string, auth authentication, sas bool) (string, error) {
errMsg := "buildCanonicalizedResource error: %s"
u, err := url.Parse(uri)
if err != nil {
return "", fmt.Errorf(errMsg, err.Error())
}
cr := bytes.NewBufferString("")
if c.accountName != StorageEmulatorAccountName || !sas {
cr.WriteString("/")
cr.WriteString(c.getCanonicalizedAccountName())
}
if len(u.Path) > 0 {
// Any portion of the CanonicalizedResource string that is derived from
// the resource's URI should be encoded exactly as it is in the URI.
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
cr.WriteString(u.EscapedPath())
}
params, err := url.ParseQuery(u.RawQuery)
if err != nil {
return "", fmt.Errorf(errMsg, err.Error())
}
// See https://github.com/Azure/azure-storage-net/blob/master/Lib/Common/Core/Util/AuthenticationUtility.cs#L277
if auth == sharedKey {
if len(params) > 0 {
cr.WriteString("\n")
keys := []string{}
for key := range params {
keys = append(keys, key)
}
sort.Strings(keys)
completeParams := []string{}
for _, key := range keys {
if len(params[key]) > 1 {
sort.Strings(params[key])
}
completeParams = append(completeParams, fmt.Sprintf("%s:%s", key, strings.Join(params[key], ",")))
}
cr.WriteString(strings.Join(completeParams, "\n"))
}
} else {
// search for "comp" parameter, if exists then add it to canonicalizedresource
if v, ok := params["comp"]; ok {
cr.WriteString("?comp=" + v[0])
}
}
return string(cr.Bytes()), nil
}
func (c *Client) getCanonicalizedAccountName() string {
// since we may be trying to access a secondary storage account, we need to
// remove the -secondary part of the storage name
return strings.TrimSuffix(c.accountName, "-secondary")
}
func buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string, auth authentication) (string, error) {
contentLength := headers[headerContentLength]
if contentLength == "0" {
contentLength = ""
}
date := headers[headerDate]
if v, ok := headers[headerXmsDate]; ok {
if auth == sharedKey || auth == sharedKeyLite {
date = ""
} else {
date = v
}
}
var canString string
switch auth {
case sharedKey:
canString = strings.Join([]string{
verb,
headers[headerContentEncoding],
headers[headerContentLanguage],
contentLength,
headers[headerContentMD5],
headers[headerContentType],
date,
headers[headerIfModifiedSince],
headers[headerIfMatch],
headers[headerIfNoneMatch],
headers[headerIfUnmodifiedSince],
headers[headerRange],
buildCanonicalizedHeader(headers),
canonicalizedResource,
}, "\n")
case sharedKeyForTable:
canString = strings.Join([]string{
verb,
headers[headerContentMD5],
headers[headerContentType],
date,
canonicalizedResource,
}, "\n")
case sharedKeyLite:
canString = strings.Join([]string{
verb,
headers[headerContentMD5],
headers[headerContentType],
date,
buildCanonicalizedHeader(headers),
canonicalizedResource,
}, "\n")
case sharedKeyLiteForTable:
canString = strings.Join([]string{
date,
canonicalizedResource,
}, "\n")
default:
return "", fmt.Errorf("%s authentication is not supported yet", auth)
}
return canString, nil
}
func buildCanonicalizedHeader(headers map[string]string) string {
cm := make(map[string]string)
for k, v := range headers {
headerName := strings.TrimSpace(strings.ToLower(k))
if strings.HasPrefix(headerName, "x-ms-") {
cm[headerName] = v
}
}
if len(cm) == 0 {
return ""
}
keys := []string{}
for key := range cm {
keys = append(keys, key)
}
sort.Strings(keys)
ch := bytes.NewBufferString("")
for _, key := range keys {
ch.WriteString(key)
ch.WriteRune(':')
ch.WriteString(cm[key])
ch.WriteRune('\n')
}
return strings.TrimSuffix(string(ch.Bytes()), "\n")
}
func (c *Client) createAuthorizationHeader(canonicalizedString string, auth authentication) string {
signature := c.computeHmac256(canonicalizedString)
var key string
switch auth {
case sharedKey, sharedKeyForTable:
key = "SharedKey"
case sharedKeyLite, sharedKeyLiteForTable:
key = "SharedKeyLite"
}
return fmt.Sprintf("%s %s:%s", key, c.getCanonicalizedAccountName(), signature)
}

View file

@ -0,0 +1,629 @@
package storage
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// A Blob is an entry in BlobListResponse.
type Blob struct {
Container *Container
Name string `xml:"Name"`
Snapshot time.Time `xml:"Snapshot"`
Properties BlobProperties `xml:"Properties"`
Metadata BlobMetadata `xml:"Metadata"`
}
// PutBlobOptions includes the options any put blob operation
// (page, block, append)
type PutBlobOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
Origin string `header:"Origin"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// BlobMetadata is a set of custom name/value pairs.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179404.aspx
type BlobMetadata map[string]string
type blobMetadataEntries struct {
Entries []blobMetadataEntry `xml:",any"`
}
type blobMetadataEntry struct {
XMLName xml.Name
Value string `xml:",chardata"`
}
// UnmarshalXML converts the xml:Metadata into Metadata map
func (bm *BlobMetadata) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var entries blobMetadataEntries
if err := d.DecodeElement(&entries, &start); err != nil {
return err
}
for _, entry := range entries.Entries {
if *bm == nil {
*bm = make(BlobMetadata)
}
(*bm)[strings.ToLower(entry.XMLName.Local)] = entry.Value
}
return nil
}
// MarshalXML implements the xml.Marshaler interface. It encodes
// metadata name/value pairs as they would appear in an Azure
// ListBlobs response.
func (bm BlobMetadata) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
entries := make([]blobMetadataEntry, 0, len(bm))
for k, v := range bm {
entries = append(entries, blobMetadataEntry{
XMLName: xml.Name{Local: http.CanonicalHeaderKey(k)},
Value: v,
})
}
return enc.EncodeElement(blobMetadataEntries{
Entries: entries,
}, start)
}
// BlobProperties contains various properties of a blob
// returned in various endpoints like ListBlobs or GetBlobProperties.
type BlobProperties struct {
LastModified TimeRFC1123 `xml:"Last-Modified"`
Etag string `xml:"Etag"`
ContentMD5 string `xml:"Content-MD5" header:"x-ms-blob-content-md5"`
ContentLength int64 `xml:"Content-Length"`
ContentType string `xml:"Content-Type" header:"x-ms-blob-content-type"`
ContentEncoding string `xml:"Content-Encoding" header:"x-ms-blob-content-encoding"`
CacheControl string `xml:"Cache-Control" header:"x-ms-blob-cache-control"`
ContentLanguage string `xml:"Cache-Language" header:"x-ms-blob-content-language"`
ContentDisposition string `xml:"Content-Disposition" header:"x-ms-blob-content-disposition"`
BlobType BlobType `xml:"x-ms-blob-blob-type"`
SequenceNumber int64 `xml:"x-ms-blob-sequence-number"`
CopyID string `xml:"CopyId"`
CopyStatus string `xml:"CopyStatus"`
CopySource string `xml:"CopySource"`
CopyProgress string `xml:"CopyProgress"`
CopyCompletionTime TimeRFC1123 `xml:"CopyCompletionTime"`
CopyStatusDescription string `xml:"CopyStatusDescription"`
LeaseStatus string `xml:"LeaseStatus"`
LeaseState string `xml:"LeaseState"`
LeaseDuration string `xml:"LeaseDuration"`
ServerEncrypted bool `xml:"ServerEncrypted"`
IncrementalCopy bool `xml:"IncrementalCopy"`
}
// BlobType defines the type of the Azure Blob.
type BlobType string
// Types of page blobs
const (
BlobTypeBlock BlobType = "BlockBlob"
BlobTypePage BlobType = "PageBlob"
BlobTypeAppend BlobType = "AppendBlob"
)
func (b *Blob) buildPath() string {
return b.Container.buildPath() + "/" + b.Name
}
// Exists returns true if a blob with given name exists on the specified
// container of the storage account.
func (b *Blob) Exists() (bool, error) {
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), nil)
headers := b.Container.bsc.client.getStandardHeaders()
resp, err := b.Container.bsc.client.exec(http.MethodHead, uri, headers, nil, b.Container.bsc.auth)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusOK, nil
}
}
return false, err
}
// GetURL gets the canonical URL to the blob with the specified name in the
// specified container. If name is not specified, the canonical URL for the entire
// container is obtained.
// This method does not create a publicly accessible URL if the blob or container
// is private and this method does not check if the blob exists.
func (b *Blob) GetURL() string {
container := b.Container.Name
if container == "" {
container = "$root"
}
return b.Container.bsc.client.getEndpoint(blobServiceName, pathForResource(container, b.Name), nil)
}
// GetBlobRangeOptions includes the options for a get blob range operation
type GetBlobRangeOptions struct {
Range *BlobRange
GetRangeContentMD5 bool
*GetBlobOptions
}
// GetBlobOptions includes the options for a get blob operation
type GetBlobOptions struct {
Timeout uint
Snapshot *time.Time
LeaseID string `header:"x-ms-lease-id"`
Origin string `header:"Origin"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// BlobRange represents the bytes range to be get
type BlobRange struct {
Start uint64
End uint64
}
func (br BlobRange) String() string {
if br.End == 0 {
return fmt.Sprintf("bytes=%d-", br.Start)
}
return fmt.Sprintf("bytes=%d-%d", br.Start, br.End)
}
// Get returns a stream to read the blob. Caller must call both Read and Close()
// to correctly close the underlying connection.
//
// See the GetRange method for use with a Range header.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Blob
func (b *Blob) Get(options *GetBlobOptions) (io.ReadCloser, error) {
rangeOptions := GetBlobRangeOptions{
GetBlobOptions: options,
}
resp, err := b.getRange(&rangeOptions)
if err != nil {
return nil, err
}
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
if err := b.writeProperties(resp.headers, true); err != nil {
return resp.body, err
}
return resp.body, nil
}
// GetRange reads the specified range of a blob to a stream. The bytesRange
// string must be in a format like "0-", "10-100" as defined in HTTP 1.1 spec.
// Caller must call both Read and Close()// to correctly close the underlying
// connection.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Blob
func (b *Blob) GetRange(options *GetBlobRangeOptions) (io.ReadCloser, error) {
resp, err := b.getRange(options)
if err != nil {
return nil, err
}
if err := checkRespCode(resp.statusCode, []int{http.StatusPartialContent}); err != nil {
return nil, err
}
// Content-Length header should not be updated, as the service returns the range length
// (which is not alwys the full blob length)
if err := b.writeProperties(resp.headers, false); err != nil {
return resp.body, err
}
return resp.body, nil
}
func (b *Blob) getRange(options *GetBlobRangeOptions) (*storageResponse, error) {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
if options.Range != nil {
headers["Range"] = options.Range.String()
if options.GetRangeContentMD5 {
headers["x-ms-range-get-content-md5"] = "true"
}
}
if options.GetBlobOptions != nil {
headers = mergeHeaders(headers, headersFromStruct(*options.GetBlobOptions))
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
}
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return nil, err
}
return resp, err
}
// SnapshotOptions includes the options for a snapshot blob operation
type SnapshotOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// CreateSnapshot creates a snapshot for a blob
// See https://msdn.microsoft.com/en-us/library/azure/ee691971.aspx
func (b *Blob) CreateSnapshot(options *SnapshotOptions) (snapshotTimestamp *time.Time, err error) {
params := url.Values{"comp": {"snapshot"}}
headers := b.Container.bsc.client.getStandardHeaders()
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil || resp == nil {
return nil, err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil {
return nil, err
}
snapshotResponse := resp.headers.Get(http.CanonicalHeaderKey("x-ms-snapshot"))
if snapshotResponse != "" {
snapshotTimestamp, err := time.Parse(time.RFC3339, snapshotResponse)
if err != nil {
return nil, err
}
return &snapshotTimestamp, nil
}
return nil, errors.New("Snapshot not created")
}
// GetBlobPropertiesOptions includes the options for a get blob properties operation
type GetBlobPropertiesOptions struct {
Timeout uint
Snapshot *time.Time
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// GetProperties provides various information about the specified blob.
// See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx
func (b *Blob) GetProperties(options *GetBlobPropertiesOptions) error {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodHead, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return err
}
return b.writeProperties(resp.headers, true)
}
func (b *Blob) writeProperties(h http.Header, includeContentLen bool) error {
var err error
contentLength := b.Properties.ContentLength
if includeContentLen {
contentLengthStr := h.Get("Content-Length")
if contentLengthStr != "" {
contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
if err != nil {
return err
}
}
}
var sequenceNum int64
sequenceNumStr := h.Get("x-ms-blob-sequence-number")
if sequenceNumStr != "" {
sequenceNum, err = strconv.ParseInt(sequenceNumStr, 0, 64)
if err != nil {
return err
}
}
lastModified, err := getTimeFromHeaders(h, "Last-Modified")
if err != nil {
return err
}
copyCompletionTime, err := getTimeFromHeaders(h, "x-ms-copy-completion-time")
if err != nil {
return err
}
b.Properties = BlobProperties{
LastModified: TimeRFC1123(*lastModified),
Etag: h.Get("Etag"),
ContentMD5: h.Get("Content-MD5"),
ContentLength: contentLength,
ContentEncoding: h.Get("Content-Encoding"),
ContentType: h.Get("Content-Type"),
ContentDisposition: h.Get("Content-Disposition"),
CacheControl: h.Get("Cache-Control"),
ContentLanguage: h.Get("Content-Language"),
SequenceNumber: sequenceNum,
CopyCompletionTime: TimeRFC1123(*copyCompletionTime),
CopyStatusDescription: h.Get("x-ms-copy-status-description"),
CopyID: h.Get("x-ms-copy-id"),
CopyProgress: h.Get("x-ms-copy-progress"),
CopySource: h.Get("x-ms-copy-source"),
CopyStatus: h.Get("x-ms-copy-status"),
BlobType: BlobType(h.Get("x-ms-blob-type")),
LeaseStatus: h.Get("x-ms-lease-status"),
LeaseState: h.Get("x-ms-lease-state"),
}
b.writeMetadata(h)
return nil
}
// SetBlobPropertiesOptions contains various properties of a blob and is an entry
// in SetProperties
type SetBlobPropertiesOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
Origin string `header:"Origin"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
SequenceNumberAction *SequenceNumberAction
RequestID string `header:"x-ms-client-request-id"`
}
// SequenceNumberAction defines how the blob's sequence number should be modified
type SequenceNumberAction string
// Options for sequence number action
const (
SequenceNumberActionMax SequenceNumberAction = "max"
SequenceNumberActionUpdate SequenceNumberAction = "update"
SequenceNumberActionIncrement SequenceNumberAction = "increment"
)
// SetProperties replaces the BlobHeaders for the specified blob.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetBlobProperties. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Blob-Properties
func (b *Blob) SetProperties(options *SetBlobPropertiesOptions) error {
params := url.Values{"comp": {"properties"}}
headers := b.Container.bsc.client.getStandardHeaders()
headers = mergeHeaders(headers, headersFromStruct(b.Properties))
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
if b.Properties.BlobType == BlobTypePage {
headers = addToHeaders(headers, "x-ms-blob-content-length", fmt.Sprintf("byte %v", b.Properties.ContentLength))
if options != nil || options.SequenceNumberAction != nil {
headers = addToHeaders(headers, "x-ms-sequence-number-action", string(*options.SequenceNumberAction))
if *options.SequenceNumberAction != SequenceNumberActionIncrement {
headers = addToHeaders(headers, "x-ms-blob-sequence-number", fmt.Sprintf("%v", b.Properties.SequenceNumber))
}
}
}
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusOK})
}
// SetBlobMetadataOptions includes the options for a set blob metadata operation
type SetBlobMetadataOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// SetMetadata replaces the metadata for the specified blob.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetBlobMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
func (b *Blob) SetMetadata(options *SetBlobMetadataOptions) error {
params := url.Values{"comp": {"metadata"}}
headers := b.Container.bsc.client.getStandardHeaders()
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusOK})
}
// GetBlobMetadataOptions includes the options for a get blob metadata operation
type GetBlobMetadataOptions struct {
Timeout uint
Snapshot *time.Time
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// GetMetadata returns all user-defined metadata for the specified blob.
//
// All metadata keys will be returned in lower case. (HTTP header
// names are case-insensitive.)
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
func (b *Blob) GetMetadata(options *GetBlobMetadataOptions) error {
params := url.Values{"comp": {"metadata"}}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return err
}
b.writeMetadata(resp.headers)
return nil
}
func (b *Blob) writeMetadata(h http.Header) {
metadata := make(map[string]string)
for k, v := range h {
// Can't trust CanonicalHeaderKey() to munge case
// reliably. "_" is allowed in identifiers:
// https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
// https://msdn.microsoft.com/library/aa664670(VS.71).aspx
// http://tools.ietf.org/html/rfc7230#section-3.2
// ...but "_" is considered invalid by
// CanonicalMIMEHeaderKey in
// https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542
// so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl".
k = strings.ToLower(k)
if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) {
continue
}
// metadata["lol"] = content of the last X-Ms-Meta-Lol header
k = k[len(userDefinedMetadataHeaderPrefix):]
metadata[k] = v[len(v)-1]
}
b.Metadata = BlobMetadata(metadata)
}
// DeleteBlobOptions includes the options for a delete blob operation
type DeleteBlobOptions struct {
Timeout uint
Snapshot *time.Time
LeaseID string `header:"x-ms-lease-id"`
DeleteSnapshots *bool
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// Delete deletes the given blob from the specified container.
// If the blob does not exists at the time of the Delete Blob operation, it
// returns error.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Blob
func (b *Blob) Delete(options *DeleteBlobOptions) error {
resp, err := b.delete(options)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusAccepted})
}
// DeleteIfExists deletes the given blob from the specified container If the
// blob is deleted with this call, returns true. Otherwise returns false.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Blob
func (b *Blob) DeleteIfExists(options *DeleteBlobOptions) (bool, error) {
resp, err := b.delete(options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
func (b *Blob) delete(options *DeleteBlobOptions) (*storageResponse, error) {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
headers = mergeHeaders(headers, headersFromStruct(*options))
if options.DeleteSnapshots != nil {
if *options.DeleteSnapshots {
headers["x-ms-delete-snapshots"] = "include"
} else {
headers["x-ms-delete-snapshots"] = "only"
}
}
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
return b.Container.bsc.client.exec(http.MethodDelete, uri, headers, nil, b.Container.bsc.auth)
}
// helper method to construct the path to either a blob or container
func pathForResource(container, name string) string {
if name != "" {
return fmt.Sprintf("/%s/%s", container, name)
}
return fmt.Sprintf("/%s", container)
}

View file

@ -0,0 +1,108 @@
package storage
import (
"errors"
"fmt"
"net/url"
"strings"
"time"
)
// GetSASURIWithSignedIPAndProtocol creates an URL to the specified blob which contains the Shared
// Access Signature with specified permissions and expiration time. Also includes signedIPRange and allowed protocols.
// If old API version is used but no signedIP is passed (ie empty string) then this should still work.
// We only populate the signedIP when it non-empty.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx
func (b *Blob) GetSASURIWithSignedIPAndProtocol(expiry time.Time, permissions string, signedIPRange string, HTTPSOnly bool) (string, error) {
var (
signedPermissions = permissions
blobURL = b.GetURL()
)
canonicalizedResource, err := b.Container.bsc.client.buildCanonicalizedResource(blobURL, b.Container.bsc.auth, true)
if err != nil {
return "", err
}
// "The canonicalizedresouce portion of the string is a canonical path to the signed resource.
// It must include the service name (blob, table, queue or file) for version 2015-02-21 or
// later, the storage account name, and the resource name, and must be URL-decoded.
// -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
// We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component).
canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1)
canonicalizedResource, err = url.QueryUnescape(canonicalizedResource)
if err != nil {
return "", err
}
signedExpiry := expiry.UTC().Format(time.RFC3339)
//If blob name is missing, resource is a container
signedResource := "c"
if len(b.Name) > 0 {
signedResource = "b"
}
protocols := ""
if HTTPSOnly {
protocols = "https"
}
stringToSign, err := blobSASStringToSign(b.Container.bsc.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions, signedIPRange, protocols)
if err != nil {
return "", err
}
sig := b.Container.bsc.client.computeHmac256(stringToSign)
sasParams := url.Values{
"sv": {b.Container.bsc.client.apiVersion},
"se": {signedExpiry},
"sr": {signedResource},
"sp": {signedPermissions},
"sig": {sig},
}
if b.Container.bsc.client.apiVersion >= "2015-04-05" {
if protocols != "" {
sasParams.Add("spr", protocols)
}
if signedIPRange != "" {
sasParams.Add("sip", signedIPRange)
}
}
sasURL, err := url.Parse(blobURL)
if err != nil {
return "", err
}
sasURL.RawQuery = sasParams.Encode()
return sasURL.String(), nil
}
// GetSASURI creates an URL to the specified blob which contains the Shared
// Access Signature with specified permissions and expiration time.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx
func (b *Blob) GetSASURI(expiry time.Time, permissions string) (string, error) {
return b.GetSASURIWithSignedIPAndProtocol(expiry, permissions, "", false)
}
func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string, signedIP string, protocols string) (string, error) {
var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string
if signedVersion >= "2015-02-21" {
canonicalizedResource = "/blob" + canonicalizedResource
}
// https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12
if signedVersion >= "2015-04-05" {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
}
// reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
if signedVersion >= "2013-08-15" {
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
}
return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15")
}

View file

@ -0,0 +1,95 @@
package storage
import (
"net/http"
"net/url"
"strconv"
)
// BlobStorageClient contains operations for Microsoft Azure Blob Storage
// Service.
type BlobStorageClient struct {
client Client
auth authentication
}
// GetServiceProperties gets the properties of your storage account's blob service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-blob-service-properties
func (b *BlobStorageClient) GetServiceProperties() (*ServiceProperties, error) {
return b.client.getServiceProperties(blobServiceName, b.auth)
}
// SetServiceProperties sets the properties of your storage account's blob service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-blob-service-properties
func (b *BlobStorageClient) SetServiceProperties(props ServiceProperties) error {
return b.client.setServiceProperties(props, blobServiceName, b.auth)
}
// ListContainersParameters defines the set of customizable parameters to make a
// List Containers call.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
type ListContainersParameters struct {
Prefix string
Marker string
Include string
MaxResults uint
Timeout uint
}
// GetContainerReference returns a Container object for the specified container name.
func (b *BlobStorageClient) GetContainerReference(name string) *Container {
return &Container{
bsc: b,
Name: name,
}
}
// ListContainers returns the list of containers in a storage account along with
// pagination token and other response details.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
func (b BlobStorageClient) ListContainers(params ListContainersParameters) (*ContainerListResponse, error) {
q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}})
uri := b.client.getEndpoint(blobServiceName, "", q)
headers := b.client.getStandardHeaders()
var out ContainerListResponse
resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
err = xmlUnmarshal(resp.body, &out)
if err != nil {
return nil, err
}
// assign our client to the newly created Container objects
for i := range out.Containers {
out.Containers[i].bsc = &b
}
return &out, err
}
func (p ListContainersParameters) getParameters() url.Values {
out := url.Values{}
if p.Prefix != "" {
out.Set("prefix", p.Prefix)
}
if p.Marker != "" {
out.Set("marker", p.Marker)
}
if p.Include != "" {
out.Set("include", p.Include)
}
if p.MaxResults != 0 {
out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
}
if p.Timeout != 0 {
out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
}
return out
}

View file

@ -0,0 +1,258 @@
package storage
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// BlockListType is used to filter out types of blocks in a Get Blocks List call
// for a block blob.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx for all
// block types.
type BlockListType string
// Filters for listing blocks in block blobs
const (
BlockListTypeAll BlockListType = "all"
BlockListTypeCommitted BlockListType = "committed"
BlockListTypeUncommitted BlockListType = "uncommitted"
)
// Maximum sizes (per REST API) for various concepts
const (
MaxBlobBlockSize = 100 * 1024 * 1024
MaxBlobPageSize = 4 * 1024 * 1024
)
// BlockStatus defines states a block for a block blob can
// be in.
type BlockStatus string
// List of statuses that can be used to refer to a block in a block list
const (
BlockStatusUncommitted BlockStatus = "Uncommitted"
BlockStatusCommitted BlockStatus = "Committed"
BlockStatusLatest BlockStatus = "Latest"
)
// Block is used to create Block entities for Put Block List
// call.
type Block struct {
ID string
Status BlockStatus
}
// BlockListResponse contains the response fields from Get Block List call.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx
type BlockListResponse struct {
XMLName xml.Name `xml:"BlockList"`
CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"`
UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"`
}
// BlockResponse contains the block information returned
// in the GetBlockListCall.
type BlockResponse struct {
Name string `xml:"Name"`
Size int64 `xml:"Size"`
}
// CreateBlockBlob initializes an empty block blob with no blocks.
//
// See CreateBlockBlobFromReader for more info on creating blobs.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
func (b *Blob) CreateBlockBlob(options *PutBlobOptions) error {
return b.CreateBlockBlobFromReader(nil, options)
}
// CreateBlockBlobFromReader initializes a block blob using data from
// reader. Size must be the number of bytes read from reader. To
// create an empty blob, use size==0 and reader==nil.
//
// Any headers set in blob.Properties or metadata in blob.Metadata
// will be set on the blob.
//
// The API rejects requests with size > 256 MiB (but this limit is not
// checked by the SDK). To write a larger blob, use CreateBlockBlob,
// PutBlock, and PutBlockList.
//
// To create a blob from scratch, call container.GetBlobReference() to
// get an empty blob, fill in blob.Properties and blob.Metadata as
// appropriate then call this method.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
func (b *Blob) CreateBlockBlobFromReader(blob io.Reader, options *PutBlobOptions) error {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypeBlock)
headers["Content-Length"] = "0"
var n int64
var err error
if blob != nil {
type lener interface {
Len() int
}
// TODO(rjeczalik): handle io.ReadSeeker, in case blob is *os.File etc.
if l, ok := blob.(lener); ok {
n = int64(l.Len())
} else {
var buf bytes.Buffer
n, err = io.Copy(&buf, blob)
if err != nil {
return err
}
blob = &buf
}
headers["Content-Length"] = strconv.FormatInt(n, 10)
}
b.Properties.ContentLength = n
headers = mergeHeaders(headers, headersFromStruct(b.Properties))
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// PutBlockOptions includes the options for a put block operation
type PutBlockOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
ContentMD5 string `header:"Content-MD5"`
RequestID string `header:"x-ms-client-request-id"`
}
// PutBlock saves the given data chunk to the specified block blob with
// given ID.
//
// The API rejects chunks larger than 100 MiB (but this limit is not
// checked by the SDK).
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block
func (b *Blob) PutBlock(blockID string, chunk []byte, options *PutBlockOptions) error {
return b.PutBlockWithLength(blockID, uint64(len(chunk)), bytes.NewReader(chunk), options)
}
// PutBlockWithLength saves the given data stream of exactly specified size to
// the block blob with given ID. It is an alternative to PutBlocks where data
// comes as stream but the length is known in advance.
//
// The API rejects requests with size > 100 MiB (but this limit is not
// checked by the SDK).
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block
func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, options *PutBlockOptions) error {
query := url.Values{
"comp": {"block"},
"blockid": {blockID},
}
headers := b.Container.bsc.client.getStandardHeaders()
headers["Content-Length"] = fmt.Sprintf("%v", size)
if options != nil {
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), query)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// PutBlockListOptions includes the options for a put block list operation
type PutBlockListOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// PutBlockList saves list of blocks to the specified block blob.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block-List
func (b *Blob) PutBlockList(blocks []Block, options *PutBlockListOptions) error {
params := url.Values{"comp": {"blocklist"}}
blockListXML := prepareBlockListRequest(blocks)
headers := b.Container.bsc.client.getStandardHeaders()
headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXML))
headers = mergeHeaders(headers, headersFromStruct(b.Properties))
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, strings.NewReader(blockListXML), b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// GetBlockListOptions includes the options for a get block list operation
type GetBlockListOptions struct {
Timeout uint
Snapshot *time.Time
LeaseID string `header:"x-ms-lease-id"`
RequestID string `header:"x-ms-client-request-id"`
}
// GetBlockList retrieves list of blocks in the specified block blob.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Block-List
func (b *Blob) GetBlockList(blockType BlockListType, options *GetBlockListOptions) (BlockListResponse, error) {
params := url.Values{
"comp": {"blocklist"},
"blocklisttype": {string(blockType)},
}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
var out BlockListResponse
resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return out, err
}
defer resp.body.Close()
err = xmlUnmarshal(resp.body, &out)
return out, err
}

View file

@ -0,0 +1,663 @@
// Package storage provides clients for Microsoft Azure Storage Services.
package storage
import (
"bufio"
"bytes"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/url"
"regexp"
"runtime"
"strings"
"time"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
)
const (
// DefaultBaseURL is the domain name used for storage requests in the
// public cloud when a default client is created.
DefaultBaseURL = "core.windows.net"
// DefaultAPIVersion is the Azure Storage API version string used when a
// basic client is created.
DefaultAPIVersion = "2016-05-31"
defaultUseHTTPS = true
// StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
StorageEmulatorAccountName = "devstoreaccount1"
// StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
blobServiceName = "blob"
tableServiceName = "table"
queueServiceName = "queue"
fileServiceName = "file"
storageEmulatorBlob = "127.0.0.1:10000"
storageEmulatorTable = "127.0.0.1:10002"
storageEmulatorQueue = "127.0.0.1:10001"
userAgentHeader = "User-Agent"
userDefinedMetadataHeaderPrefix = "x-ms-meta-"
)
var (
validStorageAccount = regexp.MustCompile("^[0-9a-z]{3,24}$")
)
// Sender sends a request
type Sender interface {
Send(*Client, *http.Request) (*http.Response, error)
}
// DefaultSender is the default sender for the client. It implements
// an automatic retry strategy.
type DefaultSender struct {
RetryAttempts int
RetryDuration time.Duration
ValidStatusCodes []int
attempts int // used for testing
}
// Send is the default retry strategy in the client
func (ds *DefaultSender) Send(c *Client, req *http.Request) (resp *http.Response, err error) {
rr := autorest.NewRetriableRequest(req)
for attempts := 0; attempts < ds.RetryAttempts; attempts++ {
err = rr.Prepare()
if err != nil {
return resp, err
}
resp, err = c.HTTPClient.Do(rr.Request())
if err != nil || !autorest.ResponseHasStatusCode(resp, ds.ValidStatusCodes...) {
return resp, err
}
autorest.DelayForBackoff(ds.RetryDuration, attempts, req.Cancel)
ds.attempts = attempts
}
ds.attempts++
return resp, err
}
// Client is the object that needs to be constructed to perform
// operations on the storage account.
type Client struct {
// HTTPClient is the http.Client used to initiate API
// requests. http.DefaultClient is used when creating a
// client.
HTTPClient *http.Client
// Sender is an interface that sends the request. Clients are
// created with a DefaultSender. The DefaultSender has an
// automatic retry strategy built in. The Sender can be customized.
Sender Sender
accountName string
accountKey []byte
useHTTPS bool
UseSharedKeyLite bool
baseURL string
apiVersion string
userAgent string
}
type storageResponse struct {
statusCode int
headers http.Header
body io.ReadCloser
}
type odataResponse struct {
storageResponse
odata odataErrorWrapper
}
// AzureStorageServiceError contains fields of the error response from
// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
// Some fields might be specific to certain calls.
type AzureStorageServiceError struct {
Code string `xml:"Code"`
Message string `xml:"Message"`
AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
QueryParameterName string `xml:"QueryParameterName"`
QueryParameterValue string `xml:"QueryParameterValue"`
Reason string `xml:"Reason"`
Lang string
StatusCode int
RequestID string
Date string
APIVersion string
}
type odataErrorMessage struct {
Lang string `json:"lang"`
Value string `json:"value"`
}
type odataError struct {
Code string `json:"code"`
Message odataErrorMessage `json:"message"`
}
type odataErrorWrapper struct {
Err odataError `json:"odata.error"`
}
// UnexpectedStatusCodeError is returned when a storage service responds with neither an error
// nor with an HTTP status code indicating success.
type UnexpectedStatusCodeError struct {
allowed []int
got int
}
func (e UnexpectedStatusCodeError) Error() string {
s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
got := s(e.got)
expected := []string{}
for _, v := range e.allowed {
expected = append(expected, s(v))
}
return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or "))
}
// Got is the actual status code returned by Azure.
func (e UnexpectedStatusCodeError) Got() int {
return e.got
}
// NewBasicClient constructs a Client with given storage service name and
// key.
func NewBasicClient(accountName, accountKey string) (Client, error) {
if accountName == StorageEmulatorAccountName {
return NewEmulatorClient()
}
return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
}
// NewBasicClientOnSovereignCloud constructs a Client with given storage service name and
// key in the referenced cloud.
func NewBasicClientOnSovereignCloud(accountName, accountKey string, env azure.Environment) (Client, error) {
if accountName == StorageEmulatorAccountName {
return NewEmulatorClient()
}
return NewClient(accountName, accountKey, env.StorageEndpointSuffix, DefaultAPIVersion, defaultUseHTTPS)
}
//NewEmulatorClient contructs a Client intended to only work with Azure
//Storage Emulator
func NewEmulatorClient() (Client, error) {
return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
}
// NewClient constructs a Client. This should be used if the caller wants
// to specify whether to use HTTPS, a specific REST API version or a custom
// storage endpoint than Azure Public Cloud.
func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
var c Client
if !IsValidStorageAccount(accountName) {
return c, fmt.Errorf("azure: account name is not valid: it must be between 3 and 24 characters, and only may contain numbers and lowercase letters: %v", accountName)
} else if accountKey == "" {
return c, fmt.Errorf("azure: account key required")
} else if blobServiceBaseURL == "" {
return c, fmt.Errorf("azure: base storage service url required")
}
key, err := base64.StdEncoding.DecodeString(accountKey)
if err != nil {
return c, fmt.Errorf("azure: malformed storage account key: %v", err)
}
c = Client{
HTTPClient: http.DefaultClient,
accountName: accountName,
accountKey: key,
useHTTPS: useHTTPS,
baseURL: blobServiceBaseURL,
apiVersion: apiVersion,
UseSharedKeyLite: false,
Sender: &DefaultSender{
RetryAttempts: 5,
ValidStatusCodes: []int{
http.StatusRequestTimeout, // 408
http.StatusInternalServerError, // 500
http.StatusBadGateway, // 502
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout, // 504
},
RetryDuration: time.Second * 5,
},
}
c.userAgent = c.getDefaultUserAgent()
return c, nil
}
// IsValidStorageAccount checks if the storage account name is valid.
// See https://docs.microsoft.com/en-us/azure/storage/storage-create-storage-account
func IsValidStorageAccount(account string) bool {
return validStorageAccount.MatchString(account)
}
func (c Client) getDefaultUserAgent() string {
return fmt.Sprintf("Go/%s (%s-%s) azure-storage-go/%s api-version/%s",
runtime.Version(),
runtime.GOARCH,
runtime.GOOS,
sdkVersion,
c.apiVersion,
)
}
// AddToUserAgent adds an extension to the current user agent
func (c *Client) AddToUserAgent(extension string) error {
if extension != "" {
c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension)
return nil
}
return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent)
}
// protectUserAgent is used in funcs that include extraheaders as a parameter.
// It prevents the User-Agent header to be overwritten, instead if it happens to
// be present, it gets added to the current User-Agent. Use it before getStandardHeaders
func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string {
if v, ok := extraheaders[userAgentHeader]; ok {
c.AddToUserAgent(v)
delete(extraheaders, userAgentHeader)
}
return extraheaders
}
func (c Client) getBaseURL(service string) *url.URL {
scheme := "http"
if c.useHTTPS {
scheme = "https"
}
host := ""
if c.accountName == StorageEmulatorAccountName {
switch service {
case blobServiceName:
host = storageEmulatorBlob
case tableServiceName:
host = storageEmulatorTable
case queueServiceName:
host = storageEmulatorQueue
}
} else {
host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
}
return &url.URL{
Scheme: scheme,
Host: host,
}
}
func (c Client) getEndpoint(service, path string, params url.Values) string {
u := c.getBaseURL(service)
// API doesn't accept path segments not starting with '/'
if !strings.HasPrefix(path, "/") {
path = fmt.Sprintf("/%v", path)
}
if c.accountName == StorageEmulatorAccountName {
path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
}
u.Path = path
u.RawQuery = params.Encode()
return u.String()
}
// GetBlobService returns a BlobStorageClient which can operate on the blob
// service of the storage account.
func (c Client) GetBlobService() BlobStorageClient {
b := BlobStorageClient{
client: c,
}
b.client.AddToUserAgent(blobServiceName)
b.auth = sharedKey
if c.UseSharedKeyLite {
b.auth = sharedKeyLite
}
return b
}
// GetQueueService returns a QueueServiceClient which can operate on the queue
// service of the storage account.
func (c Client) GetQueueService() QueueServiceClient {
q := QueueServiceClient{
client: c,
}
q.client.AddToUserAgent(queueServiceName)
q.auth = sharedKey
if c.UseSharedKeyLite {
q.auth = sharedKeyLite
}
return q
}
// GetTableService returns a TableServiceClient which can operate on the table
// service of the storage account.
func (c Client) GetTableService() TableServiceClient {
t := TableServiceClient{
client: c,
}
t.client.AddToUserAgent(tableServiceName)
t.auth = sharedKeyForTable
if c.UseSharedKeyLite {
t.auth = sharedKeyLiteForTable
}
return t
}
// GetFileService returns a FileServiceClient which can operate on the file
// service of the storage account.
func (c Client) GetFileService() FileServiceClient {
f := FileServiceClient{
client: c,
}
f.client.AddToUserAgent(fileServiceName)
f.auth = sharedKey
if c.UseSharedKeyLite {
f.auth = sharedKeyLite
}
return f
}
func (c Client) getStandardHeaders() map[string]string {
return map[string]string{
userAgentHeader: c.userAgent,
"x-ms-version": c.apiVersion,
"x-ms-date": currentTimeRfc1123Formatted(),
}
}
func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*storageResponse, error) {
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
if err != nil {
return nil, err
}
req, err := http.NewRequest(verb, url, body)
if err != nil {
return nil, errors.New("azure/storage: error creating request: " + err.Error())
}
// if a body was provided ensure that the content length was set.
// http.NewRequest() will automatically do this for a handful of types
// and for those that it doesn't we will handle here.
if body != nil && req.ContentLength < 1 {
if lr, ok := body.(*io.LimitedReader); ok {
setContentLengthFromLimitedReader(req, lr)
}
}
for k, v := range headers {
req.Header[k] = append(req.Header[k], v) // Must bypass case munging present in `Add` by using map functions directly. See https://github.com/Azure/azure-sdk-for-go/issues/645
}
resp, err := c.Sender.Send(&c, req)
if err != nil {
return nil, err
}
if resp.StatusCode >= 400 && resp.StatusCode <= 505 {
var respBody []byte
respBody, err = readAndCloseBody(resp.Body)
if err != nil {
return nil, err
}
requestID, date, version := getDebugHeaders(resp.Header)
if len(respBody) == 0 {
// no error in response body, might happen in HEAD requests
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
} else {
storageErr := AzureStorageServiceError{
StatusCode: resp.StatusCode,
RequestID: requestID,
Date: date,
APIVersion: version,
}
// response contains storage service error object, unmarshal
if resp.Header.Get("Content-Type") == "application/xml" {
errIn := serviceErrFromXML(respBody, &storageErr)
if err != nil { // error unmarshaling the error response
err = errIn
}
} else {
errIn := serviceErrFromJSON(respBody, &storageErr)
if err != nil { // error unmarshaling the error response
err = errIn
}
}
err = storageErr
}
return &storageResponse{
statusCode: resp.StatusCode,
headers: resp.Header,
body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
}, err
}
return &storageResponse{
statusCode: resp.StatusCode,
headers: resp.Header,
body: resp.Body}, nil
}
func (c Client) execInternalJSONCommon(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, *http.Request, *http.Response, error) {
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
if err != nil {
return nil, nil, nil, err
}
req, err := http.NewRequest(verb, url, body)
for k, v := range headers {
req.Header.Add(k, v)
}
resp, err := c.Sender.Send(&c, req)
if err != nil {
return nil, nil, nil, err
}
respToRet := &odataResponse{}
respToRet.body = resp.Body
respToRet.statusCode = resp.StatusCode
respToRet.headers = resp.Header
statusCode := resp.StatusCode
if statusCode >= 400 && statusCode <= 505 {
var respBody []byte
respBody, err = readAndCloseBody(resp.Body)
if err != nil {
return nil, nil, nil, err
}
requestID, date, version := getDebugHeaders(resp.Header)
if len(respBody) == 0 {
// no error in response body, might happen in HEAD requests
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
return respToRet, req, resp, err
}
// try unmarshal as odata.error json
err = json.Unmarshal(respBody, &respToRet.odata)
}
return respToRet, req, resp, err
}
func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
respToRet, _, _, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
return respToRet, err
}
func (c Client) execBatchOperationJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
// execute common query, get back generated request, response etc... for more processing.
respToRet, req, resp, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
if err != nil {
return nil, err
}
// return the OData in the case of executing batch commands.
// In this case we need to read the outer batch boundary and contents.
// Then we read the changeset information within the batch
var respBody []byte
respBody, err = readAndCloseBody(resp.Body)
if err != nil {
return nil, err
}
// outer multipart body
_, batchHeader, err := mime.ParseMediaType(resp.Header["Content-Type"][0])
if err != nil {
return nil, err
}
// batch details.
batchBoundary := batchHeader["boundary"]
batchPartBuf, changesetBoundary, err := genBatchReader(batchBoundary, respBody)
if err != nil {
return nil, err
}
// changeset details.
err = genChangesetReader(req, respToRet, batchPartBuf, changesetBoundary)
if err != nil {
return nil, err
}
return respToRet, nil
}
func genChangesetReader(req *http.Request, respToRet *odataResponse, batchPartBuf io.Reader, changesetBoundary string) error {
changesetMultiReader := multipart.NewReader(batchPartBuf, changesetBoundary)
changesetPart, err := changesetMultiReader.NextPart()
if err != nil {
return err
}
changesetPartBufioReader := bufio.NewReader(changesetPart)
changesetResp, err := http.ReadResponse(changesetPartBufioReader, req)
if err != nil {
return err
}
if changesetResp.StatusCode != http.StatusNoContent {
changesetBody, err := readAndCloseBody(changesetResp.Body)
err = json.Unmarshal(changesetBody, &respToRet.odata)
if err != nil {
return err
}
respToRet.statusCode = changesetResp.StatusCode
}
return nil
}
func genBatchReader(batchBoundary string, respBody []byte) (io.Reader, string, error) {
respBodyString := string(respBody)
respBodyReader := strings.NewReader(respBodyString)
// reading batchresponse
batchMultiReader := multipart.NewReader(respBodyReader, batchBoundary)
batchPart, err := batchMultiReader.NextPart()
if err != nil {
return nil, "", err
}
batchPartBufioReader := bufio.NewReader(batchPart)
_, changesetHeader, err := mime.ParseMediaType(batchPart.Header.Get("Content-Type"))
if err != nil {
return nil, "", err
}
changesetBoundary := changesetHeader["boundary"]
return batchPartBufioReader, changesetBoundary, nil
}
func readAndCloseBody(body io.ReadCloser) ([]byte, error) {
defer body.Close()
out, err := ioutil.ReadAll(body)
if err == io.EOF {
err = nil
}
return out, err
}
func serviceErrFromXML(body []byte, storageErr *AzureStorageServiceError) error {
if err := xml.Unmarshal(body, storageErr); err != nil {
storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
return err
}
return nil
}
func serviceErrFromJSON(body []byte, storageErr *AzureStorageServiceError) error {
odataError := odataErrorWrapper{}
if err := json.Unmarshal(body, &odataError); err != nil {
storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
return err
}
storageErr.Code = odataError.Err.Code
storageErr.Message = odataError.Err.Message.Value
storageErr.Lang = odataError.Err.Message.Lang
return nil
}
func serviceErrFromStatusCode(code int, status string, requestID, date, version string) AzureStorageServiceError {
return AzureStorageServiceError{
StatusCode: code,
Code: status,
RequestID: requestID,
Date: date,
APIVersion: version,
Message: "no response body was available for error status code",
}
}
func (e AzureStorageServiceError) Error() string {
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestInitiated=%s, RequestId=%s, API Version=%s, QueryParameterName=%s, QueryParameterValue=%s",
e.StatusCode, e.Code, e.Message, e.Date, e.RequestID, e.APIVersion, e.QueryParameterName, e.QueryParameterValue)
}
// checkRespCode returns UnexpectedStatusError if the given response code is not
// one of the allowed status codes; otherwise nil.
func checkRespCode(respCode int, allowed []int) error {
for _, v := range allowed {
if respCode == v {
return nil
}
}
return UnexpectedStatusCodeError{allowed, respCode}
}
func (c Client) addMetadataToHeaders(h map[string]string, metadata map[string]string) map[string]string {
metadata = c.protectUserAgent(metadata)
for k, v := range metadata {
h[userDefinedMetadataHeaderPrefix+k] = v
}
return h
}
func getDebugHeaders(h http.Header) (requestID, date, version string) {
requestID = h.Get("x-ms-request-id")
version = h.Get("x-ms-version")
date = h.Get("Date")
return
}

View file

@ -0,0 +1,453 @@
package storage
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// Container represents an Azure container.
type Container struct {
bsc *BlobStorageClient
Name string `xml:"Name"`
Properties ContainerProperties `xml:"Properties"`
Metadata map[string]string
}
func (c *Container) buildPath() string {
return fmt.Sprintf("/%s", c.Name)
}
// ContainerProperties contains various properties of a container returned from
// various endpoints like ListContainers.
type ContainerProperties struct {
LastModified string `xml:"Last-Modified"`
Etag string `xml:"Etag"`
LeaseStatus string `xml:"LeaseStatus"`
LeaseState string `xml:"LeaseState"`
LeaseDuration string `xml:"LeaseDuration"`
}
// ContainerListResponse contains the response fields from
// ListContainers call.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
type ContainerListResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
Xmlns string `xml:"xmlns,attr"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker"`
MaxResults int64 `xml:"MaxResults"`
Containers []Container `xml:"Containers>Container"`
}
// BlobListResponse contains the response fields from ListBlobs call.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
type BlobListResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
Xmlns string `xml:"xmlns,attr"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker"`
MaxResults int64 `xml:"MaxResults"`
Blobs []Blob `xml:"Blobs>Blob"`
// BlobPrefix is used to traverse blobs as if it were a file system.
// It is returned if ListBlobsParameters.Delimiter is specified.
// The list here can be thought of as "folders" that may contain
// other folders or blobs.
BlobPrefixes []string `xml:"Blobs>BlobPrefix>Name"`
// Delimiter is used to traverse blobs as if it were a file system.
// It is returned if ListBlobsParameters.Delimiter is specified.
Delimiter string `xml:"Delimiter"`
}
// IncludeBlobDataset has options to include in a list blobs operation
type IncludeBlobDataset struct {
Snapshots bool
Metadata bool
UncommittedBlobs bool
Copy bool
}
// ListBlobsParameters defines the set of customizable
// parameters to make a List Blobs call.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
type ListBlobsParameters struct {
Prefix string
Delimiter string
Marker string
Include *IncludeBlobDataset
MaxResults uint
Timeout uint
RequestID string
}
func (p ListBlobsParameters) getParameters() url.Values {
out := url.Values{}
if p.Prefix != "" {
out.Set("prefix", p.Prefix)
}
if p.Delimiter != "" {
out.Set("delimiter", p.Delimiter)
}
if p.Marker != "" {
out.Set("marker", p.Marker)
}
if p.Include != nil {
include := []string{}
include = addString(include, p.Include.Snapshots, "snapshots")
include = addString(include, p.Include.Metadata, "metadata")
include = addString(include, p.Include.UncommittedBlobs, "uncommittedblobs")
include = addString(include, p.Include.Copy, "copy")
fullInclude := strings.Join(include, ",")
out.Set("include", fullInclude)
}
if p.MaxResults != 0 {
out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
}
if p.Timeout != 0 {
out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
}
return out
}
func addString(datasets []string, include bool, text string) []string {
if include {
datasets = append(datasets, text)
}
return datasets
}
// ContainerAccessType defines the access level to the container from a public
// request.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx and "x-ms-
// blob-public-access" header.
type ContainerAccessType string
// Access options for containers
const (
ContainerAccessTypePrivate ContainerAccessType = ""
ContainerAccessTypeBlob ContainerAccessType = "blob"
ContainerAccessTypeContainer ContainerAccessType = "container"
)
// ContainerAccessPolicy represents each access policy in the container ACL.
type ContainerAccessPolicy struct {
ID string
StartTime time.Time
ExpiryTime time.Time
CanRead bool
CanWrite bool
CanDelete bool
}
// ContainerPermissions represents the container ACLs.
type ContainerPermissions struct {
AccessType ContainerAccessType
AccessPolicies []ContainerAccessPolicy
}
// ContainerAccessHeader references header used when setting/getting container ACL
const (
ContainerAccessHeader string = "x-ms-blob-public-access"
)
// GetBlobReference returns a Blob object for the specified blob name.
func (c *Container) GetBlobReference(name string) *Blob {
return &Blob{
Container: c,
Name: name,
}
}
// CreateContainerOptions includes the options for a create container operation
type CreateContainerOptions struct {
Timeout uint
Access ContainerAccessType `header:"x-ms-blob-public-access"`
RequestID string `header:"x-ms-client-request-id"`
}
// Create creates a blob container within the storage account
// with given name and access level. Returns error if container already exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Container
func (c *Container) Create(options *CreateContainerOptions) error {
resp, err := c.create(options)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// CreateIfNotExists creates a blob container if it does not exist. Returns
// true if container is newly created or false if container already exists.
func (c *Container) CreateIfNotExists(options *CreateContainerOptions) (bool, error) {
resp, err := c.create(options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict {
return resp.statusCode == http.StatusCreated, nil
}
}
return false, err
}
func (c *Container) create(options *CreateContainerOptions) (*storageResponse, error) {
query := url.Values{"restype": {"container"}}
headers := c.bsc.client.getStandardHeaders()
headers = c.bsc.client.addMetadataToHeaders(headers, c.Metadata)
if options != nil {
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), query)
return c.bsc.client.exec(http.MethodPut, uri, headers, nil, c.bsc.auth)
}
// Exists returns true if a container with given name exists
// on the storage account, otherwise returns false.
func (c *Container) Exists() (bool, error) {
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), url.Values{"restype": {"container"}})
headers := c.bsc.client.getStandardHeaders()
resp, err := c.bsc.client.exec(http.MethodHead, uri, headers, nil, c.bsc.auth)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusOK, nil
}
}
return false, err
}
// SetContainerPermissionOptions includes options for a set container permissions operation
type SetContainerPermissionOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
RequestID string `header:"x-ms-client-request-id"`
}
// SetPermissions sets up container permissions
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Container-ACL
func (c *Container) SetPermissions(permissions ContainerPermissions, options *SetContainerPermissionOptions) error {
body, length, err := generateContainerACLpayload(permissions.AccessPolicies)
if err != nil {
return err
}
params := url.Values{
"restype": {"container"},
"comp": {"acl"},
}
headers := c.bsc.client.getStandardHeaders()
headers = addToHeaders(headers, ContainerAccessHeader, string(permissions.AccessType))
headers["Content-Length"] = strconv.Itoa(length)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
resp, err := c.bsc.client.exec(http.MethodPut, uri, headers, body, c.bsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return errors.New("Unable to set permissions")
}
return nil
}
// GetContainerPermissionOptions includes options for a get container permissions operation
type GetContainerPermissionOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
RequestID string `header:"x-ms-client-request-id"`
}
// GetPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx
// If timeout is 0 then it will not be passed to Azure
// leaseID will only be passed to Azure if populated
func (c *Container) GetPermissions(options *GetContainerPermissionOptions) (*ContainerPermissions, error) {
params := url.Values{
"restype": {"container"},
"comp": {"acl"},
}
headers := c.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
var ap AccessPolicy
err = xmlUnmarshal(resp.body, &ap.SignedIdentifiersList)
if err != nil {
return nil, err
}
return buildAccessPolicy(ap, &resp.headers), nil
}
func buildAccessPolicy(ap AccessPolicy, headers *http.Header) *ContainerPermissions {
// containerAccess. Blob, Container, empty
containerAccess := headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader))
permissions := ContainerPermissions{
AccessType: ContainerAccessType(containerAccess),
AccessPolicies: []ContainerAccessPolicy{},
}
for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
capd := ContainerAccessPolicy{
ID: policy.ID,
StartTime: policy.AccessPolicy.StartTime,
ExpiryTime: policy.AccessPolicy.ExpiryTime,
}
capd.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
capd.CanWrite = updatePermissions(policy.AccessPolicy.Permission, "w")
capd.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d")
permissions.AccessPolicies = append(permissions.AccessPolicies, capd)
}
return &permissions
}
// DeleteContainerOptions includes options for a delete container operation
type DeleteContainerOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
RequestID string `header:"x-ms-client-request-id"`
}
// Delete deletes the container with given name on the storage
// account. If the container does not exist returns error.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-container
func (c *Container) Delete(options *DeleteContainerOptions) error {
resp, err := c.delete(options)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusAccepted})
}
// DeleteIfExists deletes the container with given name on the storage
// account if it exists. Returns true if container is deleted with this call, or
// false if the container did not exist at the time of the Delete Container
// operation.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-container
func (c *Container) DeleteIfExists(options *DeleteContainerOptions) (bool, error) {
resp, err := c.delete(options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
func (c *Container) delete(options *DeleteContainerOptions) (*storageResponse, error) {
query := url.Values{"restype": {"container"}}
headers := c.bsc.client.getStandardHeaders()
if options != nil {
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), query)
return c.bsc.client.exec(http.MethodDelete, uri, headers, nil, c.bsc.auth)
}
// ListBlobs returns an object that contains list of blobs in the container,
// pagination token and other information in the response of List Blobs call.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Blobs
func (c *Container) ListBlobs(params ListBlobsParameters) (BlobListResponse, error) {
q := mergeParams(params.getParameters(), url.Values{
"restype": {"container"},
"comp": {"list"}},
)
uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q)
headers := c.bsc.client.getStandardHeaders()
headers = addToHeaders(headers, "x-ms-client-request-id", params.RequestID)
var out BlobListResponse
resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
if err != nil {
return out, err
}
defer resp.body.Close()
err = xmlUnmarshal(resp.body, &out)
for i := range out.Blobs {
out.Blobs[i].Container = c
}
return out, err
}
func generateContainerACLpayload(policies []ContainerAccessPolicy) (io.Reader, int, error) {
sil := SignedIdentifiers{
SignedIdentifiers: []SignedIdentifier{},
}
for _, capd := range policies {
permission := capd.generateContainerPermissions()
signedIdentifier := convertAccessPolicyToXMLStructs(capd.ID, capd.StartTime, capd.ExpiryTime, permission)
sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
}
return xmlMarshal(sil)
}
func (capd *ContainerAccessPolicy) generateContainerPermissions() (permissions string) {
// generate the permissions string (rwd).
// still want the end user API to have bool flags.
permissions = ""
if capd.CanRead {
permissions += "r"
}
if capd.CanWrite {
permissions += "w"
}
if capd.CanDelete {
permissions += "d"
}
return permissions
}

View file

@ -0,0 +1,223 @@
package storage
import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
)
const (
blobCopyStatusPending = "pending"
blobCopyStatusSuccess = "success"
blobCopyStatusAborted = "aborted"
blobCopyStatusFailed = "failed"
)
// CopyOptions includes the options for a copy blob operation
type CopyOptions struct {
Timeout uint
Source CopyOptionsConditions
Destiny CopyOptionsConditions
RequestID string
}
// IncrementalCopyOptions includes the options for an incremental copy blob operation
type IncrementalCopyOptions struct {
Timeout uint
Destination IncrementalCopyOptionsConditions
RequestID string
}
// CopyOptionsConditions includes some conditional options in a copy blob operation
type CopyOptionsConditions struct {
LeaseID string
IfModifiedSince *time.Time
IfUnmodifiedSince *time.Time
IfMatch string
IfNoneMatch string
}
// IncrementalCopyOptionsConditions includes some conditional options in a copy blob operation
type IncrementalCopyOptionsConditions struct {
IfModifiedSince *time.Time
IfUnmodifiedSince *time.Time
IfMatch string
IfNoneMatch string
}
// Copy starts a blob copy operation and waits for the operation to
// complete. sourceBlob parameter must be a canonical URL to the blob (can be
// obtained using the GetURL method.) There is no SLA on blob copy and therefore
// this helper method works faster on smaller files.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob
func (b *Blob) Copy(sourceBlob string, options *CopyOptions) error {
copyID, err := b.StartCopy(sourceBlob, options)
if err != nil {
return err
}
return b.WaitForCopy(copyID)
}
// StartCopy starts a blob copy operation.
// sourceBlob parameter must be a canonical URL to the blob (can be
// obtained using the GetURL method.)
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob
func (b *Blob) StartCopy(sourceBlob string, options *CopyOptions) (string, error) {
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-copy-source"] = sourceBlob
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
// source
headers = addToHeaders(headers, "x-ms-source-lease-id", options.Source.LeaseID)
headers = addTimeToHeaders(headers, "x-ms-source-if-modified-since", options.Source.IfModifiedSince)
headers = addTimeToHeaders(headers, "x-ms-source-if-unmodified-since", options.Source.IfUnmodifiedSince)
headers = addToHeaders(headers, "x-ms-source-if-match", options.Source.IfMatch)
headers = addToHeaders(headers, "x-ms-source-if-none-match", options.Source.IfNoneMatch)
//destiny
headers = addToHeaders(headers, "x-ms-lease-id", options.Destiny.LeaseID)
headers = addTimeToHeaders(headers, "x-ms-if-modified-since", options.Destiny.IfModifiedSince)
headers = addTimeToHeaders(headers, "x-ms-if-unmodified-since", options.Destiny.IfUnmodifiedSince)
headers = addToHeaders(headers, "x-ms-if-match", options.Destiny.IfMatch)
headers = addToHeaders(headers, "x-ms-if-none-match", options.Destiny.IfNoneMatch)
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return "", err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted, http.StatusCreated}); err != nil {
return "", err
}
copyID := resp.headers.Get("x-ms-copy-id")
if copyID == "" {
return "", errors.New("Got empty copy id header")
}
return copyID, nil
}
// AbortCopyOptions includes the options for an abort blob operation
type AbortCopyOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
RequestID string `header:"x-ms-client-request-id"`
}
// AbortCopy aborts a BlobCopy which has already been triggered by the StartBlobCopy function.
// copyID is generated from StartBlobCopy function.
// currentLeaseID is required IF the destination blob has an active lease on it.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Abort-Copy-Blob
func (b *Blob) AbortCopy(copyID string, options *AbortCopyOptions) error {
params := url.Values{
"comp": {"copy"},
"copyid": {copyID},
}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-copy-action"] = "abort"
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// WaitForCopy loops until a BlobCopy operation is completed (or fails with error)
func (b *Blob) WaitForCopy(copyID string) error {
for {
err := b.GetProperties(nil)
if err != nil {
return err
}
if b.Properties.CopyID != copyID {
return errBlobCopyIDMismatch
}
switch b.Properties.CopyStatus {
case blobCopyStatusSuccess:
return nil
case blobCopyStatusPending:
continue
case blobCopyStatusAborted:
return errBlobCopyAborted
case blobCopyStatusFailed:
return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", b.Properties.CopyID, b.Properties.CopyStatusDescription)
default:
return fmt.Errorf("storage: unhandled blob copy status: '%s'", b.Properties.CopyStatus)
}
}
}
// IncrementalCopyBlob copies a snapshot of a source blob and copies to referring blob
// sourceBlob parameter must be a valid snapshot URL of the original blob.
// THe original blob mut be public, or use a Shared Access Signature.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/incremental-copy-blob .
func (b *Blob) IncrementalCopyBlob(sourceBlobURL string, snapshotTime time.Time, options *IncrementalCopyOptions) (string, error) {
params := url.Values{"comp": {"incrementalcopy"}}
// need formatting to 7 decimal places so it's friendly to Windows and *nix
snapshotTimeFormatted := snapshotTime.Format("2006-01-02T15:04:05.0000000Z")
u, err := url.Parse(sourceBlobURL)
if err != nil {
return "", err
}
query := u.Query()
query.Add("snapshot", snapshotTimeFormatted)
encodedQuery := query.Encode()
encodedQuery = strings.Replace(encodedQuery, "%3A", ":", -1)
u.RawQuery = encodedQuery
snapshotURL := u.String()
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-copy-source"] = snapshotURL
if options != nil {
addTimeout(params, options.Timeout)
headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
headers = addTimeToHeaders(headers, "x-ms-if-modified-since", options.Destination.IfModifiedSince)
headers = addTimeToHeaders(headers, "x-ms-if-unmodified-since", options.Destination.IfUnmodifiedSince)
headers = addToHeaders(headers, "x-ms-if-match", options.Destination.IfMatch)
headers = addToHeaders(headers, "x-ms-if-none-match", options.Destination.IfNoneMatch)
}
// get URI of destination blob
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return "", err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted}); err != nil {
return "", err
}
copyID := resp.headers.Get("x-ms-copy-id")
if copyID == "" {
return "", errors.New("Got empty copy id header")
}
return copyID, nil
}

View file

@ -0,0 +1,224 @@
package storage
import (
"encoding/xml"
"net/http"
"net/url"
"sync"
)
// Directory represents a directory on a share.
type Directory struct {
fsc *FileServiceClient
Metadata map[string]string
Name string `xml:"Name"`
parent *Directory
Properties DirectoryProperties
share *Share
}
// DirectoryProperties contains various properties of a directory.
type DirectoryProperties struct {
LastModified string `xml:"Last-Modified"`
Etag string `xml:"Etag"`
}
// ListDirsAndFilesParameters defines the set of customizable parameters to
// make a List Files and Directories call.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
type ListDirsAndFilesParameters struct {
Prefix string
Marker string
MaxResults uint
Timeout uint
}
// DirsAndFilesListResponse contains the response fields from
// a List Files and Directories call.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
type DirsAndFilesListResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
Xmlns string `xml:"xmlns,attr"`
Marker string `xml:"Marker"`
MaxResults int64 `xml:"MaxResults"`
Directories []Directory `xml:"Entries>Directory"`
Files []File `xml:"Entries>File"`
NextMarker string `xml:"NextMarker"`
}
// builds the complete directory path for this directory object.
func (d *Directory) buildPath() string {
path := ""
current := d
for current.Name != "" {
path = "/" + current.Name + path
current = current.parent
}
return d.share.buildPath() + path
}
// Create this directory in the associated share.
// If a directory with the same name already exists, the operation fails.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Directory
func (d *Directory) Create(options *FileRequestOptions) error {
// if this is the root directory exit early
if d.parent == nil {
return nil
}
params := prepareOptions(options)
headers, err := d.fsc.createResource(d.buildPath(), resourceDirectory, params, mergeMDIntoExtraHeaders(d.Metadata, nil), []int{http.StatusCreated})
if err != nil {
return err
}
d.updateEtagAndLastModified(headers)
return nil
}
// CreateIfNotExists creates this directory under the associated share if the
// directory does not exists. Returns true if the directory is newly created or
// false if the directory already exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Directory
func (d *Directory) CreateIfNotExists(options *FileRequestOptions) (bool, error) {
// if this is the root directory exit early
if d.parent == nil {
return false, nil
}
params := prepareOptions(options)
resp, err := d.fsc.createResourceNoClose(d.buildPath(), resourceDirectory, params, nil)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict {
if resp.statusCode == http.StatusCreated {
d.updateEtagAndLastModified(resp.headers)
return true, nil
}
return false, d.FetchAttributes(nil)
}
}
return false, err
}
// Delete removes this directory. It must be empty in order to be deleted.
// If the directory does not exist the operation fails.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Directory
func (d *Directory) Delete(options *FileRequestOptions) error {
return d.fsc.deleteResource(d.buildPath(), resourceDirectory, options)
}
// DeleteIfExists removes this directory if it exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Directory
func (d *Directory) DeleteIfExists(options *FileRequestOptions) (bool, error) {
resp, err := d.fsc.deleteResourceNoClose(d.buildPath(), resourceDirectory, options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
// Exists returns true if this directory exists.
func (d *Directory) Exists() (bool, error) {
exists, headers, err := d.fsc.resourceExists(d.buildPath(), resourceDirectory)
if exists {
d.updateEtagAndLastModified(headers)
}
return exists, err
}
// FetchAttributes retrieves metadata for this directory.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-directory-properties
func (d *Directory) FetchAttributes(options *FileRequestOptions) error {
params := prepareOptions(options)
headers, err := d.fsc.getResourceHeaders(d.buildPath(), compNone, resourceDirectory, params, http.MethodHead)
if err != nil {
return err
}
d.updateEtagAndLastModified(headers)
d.Metadata = getMetadataFromHeaders(headers)
return nil
}
// GetDirectoryReference returns a child Directory object for this directory.
func (d *Directory) GetDirectoryReference(name string) *Directory {
return &Directory{
fsc: d.fsc,
Name: name,
parent: d,
share: d.share,
}
}
// GetFileReference returns a child File object for this directory.
func (d *Directory) GetFileReference(name string) *File {
return &File{
fsc: d.fsc,
Name: name,
parent: d,
share: d.share,
mutex: &sync.Mutex{},
}
}
// ListDirsAndFiles returns a list of files and directories under this directory.
// It also contains a pagination token and other response details.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
func (d *Directory) ListDirsAndFiles(params ListDirsAndFilesParameters) (*DirsAndFilesListResponse, error) {
q := mergeParams(params.getParameters(), getURLInitValues(compList, resourceDirectory))
resp, err := d.fsc.listContent(d.buildPath(), q, nil)
if err != nil {
return nil, err
}
defer resp.body.Close()
var out DirsAndFilesListResponse
err = xmlUnmarshal(resp.body, &out)
return &out, err
}
// SetMetadata replaces the metadata for this directory.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetDirectoryMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Directory-Metadata
func (d *Directory) SetMetadata(options *FileRequestOptions) error {
headers, err := d.fsc.setResourceHeaders(d.buildPath(), compMetadata, resourceDirectory, mergeMDIntoExtraHeaders(d.Metadata, nil), options)
if err != nil {
return err
}
d.updateEtagAndLastModified(headers)
return nil
}
// updates Etag and last modified date
func (d *Directory) updateEtagAndLastModified(headers http.Header) {
d.Properties.Etag = headers.Get("Etag")
d.Properties.LastModified = headers.Get("Last-Modified")
}
// URL gets the canonical URL to this directory.
// This method does not create a publicly accessible URL if the directory
// is private and this method does not check if the directory exists.
func (d *Directory) URL() string {
return d.fsc.client.getEndpoint(fileServiceName, d.buildPath(), url.Values{})
}

View file

@ -0,0 +1,439 @@
package storage
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/satori/uuid"
)
// Annotating as secure for gas scanning
/* #nosec */
const (
partitionKeyNode = "PartitionKey"
rowKeyNode = "RowKey"
etagErrorTemplate = "Etag didn't match: %v"
)
var (
errEmptyPayload = errors.New("Empty payload is not a valid metadata level for this operation")
errNilPreviousResult = errors.New("The previous results page is nil")
errNilNextLink = errors.New("There are no more pages in this query results")
)
// Entity represents an entity inside an Azure table.
type Entity struct {
Table *Table
PartitionKey string
RowKey string
TimeStamp time.Time
OdataMetadata string
OdataType string
OdataID string
OdataEtag string
OdataEditLink string
Properties map[string]interface{}
}
// GetEntityReference returns an Entity object with the specified
// partition key and row key.
func (t *Table) GetEntityReference(partitionKey, rowKey string) *Entity {
return &Entity{
PartitionKey: partitionKey,
RowKey: rowKey,
Table: t,
}
}
// EntityOptions includes options for entity operations.
type EntityOptions struct {
Timeout uint
RequestID string `header:"x-ms-client-request-id"`
}
// GetEntityOptions includes options for a get entity operation
type GetEntityOptions struct {
Select []string
RequestID string `header:"x-ms-client-request-id"`
}
// Get gets the referenced entity. Which properties to get can be
// specified using the select option.
// See:
// https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
// https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/querying-tables-and-entities
func (e *Entity) Get(timeout uint, ml MetadataLevel, options *GetEntityOptions) error {
if ml == EmptyPayload {
return errEmptyPayload
}
// RowKey and PartitionKey could be lost if not included in the query
// As those are the entity identifiers, it is best if they are not lost
rk := e.RowKey
pk := e.PartitionKey
query := url.Values{
"timeout": {strconv.FormatUint(uint64(timeout), 10)},
}
headers := e.Table.tsc.client.getStandardHeaders()
headers[headerAccept] = string(ml)
if options != nil {
if len(options.Select) > 0 {
query.Add("$select", strings.Join(options.Select, ","))
}
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
resp, err := e.Table.tsc.client.exec(http.MethodGet, uri, headers, nil, e.Table.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return err
}
respBody, err := ioutil.ReadAll(resp.body)
if err != nil {
return err
}
err = json.Unmarshal(respBody, e)
if err != nil {
return err
}
e.PartitionKey = pk
e.RowKey = rk
return nil
}
// Insert inserts the referenced entity in its table.
// The function fails if there is an entity with the same
// PartitionKey and RowKey in the table.
// ml determines the level of detail of metadata in the operation response,
// or no data at all.
// See: https://docs.microsoft.com/rest/api/storageservices/fileservices/insert-entity
func (e *Entity) Insert(ml MetadataLevel, options *EntityOptions) error {
query, headers := options.getParameters()
headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
body, err := json.Marshal(e)
if err != nil {
return err
}
headers = addBodyRelatedHeaders(headers, len(body))
headers = addReturnContentHeaders(headers, ml)
uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.Table.buildPath(), query)
resp, err := e.Table.tsc.client.exec(http.MethodPost, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
if err != nil {
return err
}
defer resp.body.Close()
data, err := ioutil.ReadAll(resp.body)
if err != nil {
return err
}
if ml != EmptyPayload {
if err = checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil {
return err
}
if err = e.UnmarshalJSON(data); err != nil {
return err
}
} else {
if err = checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
}
return nil
}
// Update updates the contents of an entity. The function fails if there is no entity
// with the same PartitionKey and RowKey in the table or if the ETag is different
// than the one in Azure.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/update-entity2
func (e *Entity) Update(force bool, options *EntityOptions) error {
return e.updateMerge(force, http.MethodPut, options)
}
// Merge merges the contents of entity specified with PartitionKey and RowKey
// with the content specified in Properties.
// The function fails if there is no entity with the same PartitionKey and
// RowKey in the table or if the ETag is different than the one in Azure.
// Read more: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/merge-entity
func (e *Entity) Merge(force bool, options *EntityOptions) error {
return e.updateMerge(force, "MERGE", options)
}
// Delete deletes the entity.
// The function fails if there is no entity with the same PartitionKey and
// RowKey in the table or if the ETag is different than the one in Azure.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-entity1
func (e *Entity) Delete(force bool, options *EntityOptions) error {
query, headers := options.getParameters()
headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
headers = addIfMatchHeader(headers, force, e.OdataEtag)
headers = addReturnContentHeaders(headers, EmptyPayload)
uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
resp, err := e.Table.tsc.client.exec(http.MethodDelete, uri, headers, nil, e.Table.tsc.auth)
if err != nil {
if resp.statusCode == http.StatusPreconditionFailed {
return fmt.Errorf(etagErrorTemplate, err)
}
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return e.updateTimestamp(resp.headers)
}
// InsertOrReplace inserts an entity or replaces the existing one.
// Read more: https://docs.microsoft.com/rest/api/storageservices/fileservices/insert-or-replace-entity
func (e *Entity) InsertOrReplace(options *EntityOptions) error {
return e.insertOr(http.MethodPut, options)
}
// InsertOrMerge inserts an entity or merges the existing one.
// Read more: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/insert-or-merge-entity
func (e *Entity) InsertOrMerge(options *EntityOptions) error {
return e.insertOr("MERGE", options)
}
func (e *Entity) buildPath() string {
return fmt.Sprintf("%s(PartitionKey='%s', RowKey='%s')", e.Table.buildPath(), e.PartitionKey, e.RowKey)
}
// MarshalJSON is a custom marshaller for entity
func (e *Entity) MarshalJSON() ([]byte, error) {
completeMap := map[string]interface{}{}
completeMap[partitionKeyNode] = e.PartitionKey
completeMap[rowKeyNode] = e.RowKey
for k, v := range e.Properties {
typeKey := strings.Join([]string{k, OdataTypeSuffix}, "")
switch t := v.(type) {
case []byte:
completeMap[typeKey] = OdataBinary
completeMap[k] = string(t)
case time.Time:
completeMap[typeKey] = OdataDateTime
completeMap[k] = t.Format(time.RFC3339Nano)
case uuid.UUID:
completeMap[typeKey] = OdataGUID
completeMap[k] = t.String()
case int64:
completeMap[typeKey] = OdataInt64
completeMap[k] = fmt.Sprintf("%v", v)
default:
completeMap[k] = v
}
if strings.HasSuffix(k, OdataTypeSuffix) {
if !(completeMap[k] == OdataBinary ||
completeMap[k] == OdataDateTime ||
completeMap[k] == OdataGUID ||
completeMap[k] == OdataInt64) {
return nil, fmt.Errorf("Odata.type annotation %v value is not valid", k)
}
valueKey := strings.TrimSuffix(k, OdataTypeSuffix)
if _, ok := completeMap[valueKey]; !ok {
return nil, fmt.Errorf("Odata.type annotation %v defined without value defined", k)
}
}
}
return json.Marshal(completeMap)
}
// UnmarshalJSON is a custom unmarshaller for entities
func (e *Entity) UnmarshalJSON(data []byte) error {
errorTemplate := "Deserializing error: %v"
props := map[string]interface{}{}
err := json.Unmarshal(data, &props)
if err != nil {
return err
}
// deselialize metadata
e.OdataMetadata = stringFromMap(props, "odata.metadata")
e.OdataType = stringFromMap(props, "odata.type")
e.OdataID = stringFromMap(props, "odata.id")
e.OdataEtag = stringFromMap(props, "odata.etag")
e.OdataEditLink = stringFromMap(props, "odata.editLink")
e.PartitionKey = stringFromMap(props, partitionKeyNode)
e.RowKey = stringFromMap(props, rowKeyNode)
// deserialize timestamp
timeStamp, ok := props["Timestamp"]
if ok {
str, ok := timeStamp.(string)
if !ok {
return fmt.Errorf(errorTemplate, "Timestamp casting error")
}
t, err := time.Parse(time.RFC3339Nano, str)
if err != nil {
return fmt.Errorf(errorTemplate, err)
}
e.TimeStamp = t
}
delete(props, "Timestamp")
delete(props, "Timestamp@odata.type")
// deserialize entity (user defined fields)
for k, v := range props {
if strings.HasSuffix(k, OdataTypeSuffix) {
valueKey := strings.TrimSuffix(k, OdataTypeSuffix)
str, ok := props[valueKey].(string)
if !ok {
return fmt.Errorf(errorTemplate, fmt.Sprintf("%v casting error", v))
}
switch v {
case OdataBinary:
props[valueKey] = []byte(str)
case OdataDateTime:
t, err := time.Parse("2006-01-02T15:04:05Z", str)
if err != nil {
return fmt.Errorf(errorTemplate, err)
}
props[valueKey] = t
case OdataGUID:
props[valueKey] = uuid.FromStringOrNil(str)
case OdataInt64:
i, err := strconv.ParseInt(str, 10, 64)
if err != nil {
return fmt.Errorf(errorTemplate, err)
}
props[valueKey] = i
default:
return fmt.Errorf(errorTemplate, fmt.Sprintf("%v is not supported", v))
}
delete(props, k)
}
}
e.Properties = props
return nil
}
func getAndDelete(props map[string]interface{}, key string) interface{} {
if value, ok := props[key]; ok {
delete(props, key)
return value
}
return nil
}
func addIfMatchHeader(h map[string]string, force bool, etag string) map[string]string {
if force {
h[headerIfMatch] = "*"
} else {
h[headerIfMatch] = etag
}
return h
}
// updates Etag and timestamp
func (e *Entity) updateEtagAndTimestamp(headers http.Header) error {
e.OdataEtag = headers.Get(headerEtag)
return e.updateTimestamp(headers)
}
func (e *Entity) updateTimestamp(headers http.Header) error {
str := headers.Get(headerDate)
t, err := time.Parse(time.RFC1123, str)
if err != nil {
return fmt.Errorf("Update timestamp error: %v", err)
}
e.TimeStamp = t
return nil
}
func (e *Entity) insertOr(verb string, options *EntityOptions) error {
query, headers := options.getParameters()
headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
body, err := json.Marshal(e)
if err != nil {
return err
}
headers = addBodyRelatedHeaders(headers, len(body))
headers = addReturnContentHeaders(headers, EmptyPayload)
uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
resp, err := e.Table.tsc.client.exec(verb, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return e.updateEtagAndTimestamp(resp.headers)
}
func (e *Entity) updateMerge(force bool, verb string, options *EntityOptions) error {
query, headers := options.getParameters()
headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
body, err := json.Marshal(e)
if err != nil {
return err
}
headers = addBodyRelatedHeaders(headers, len(body))
headers = addIfMatchHeader(headers, force, e.OdataEtag)
headers = addReturnContentHeaders(headers, EmptyPayload)
uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
resp, err := e.Table.tsc.client.exec(verb, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
if err != nil {
if resp.statusCode == http.StatusPreconditionFailed {
return fmt.Errorf(etagErrorTemplate, err)
}
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return e.updateEtagAndTimestamp(resp.headers)
}
func stringFromMap(props map[string]interface{}, key string) string {
value := getAndDelete(props, key)
if value != nil {
return value.(string)
}
return ""
}
func (options *EntityOptions) getParameters() (url.Values, map[string]string) {
query := url.Values{}
headers := map[string]string{}
if options != nil {
query = addTimeout(query, options.Timeout)
headers = headersFromStruct(*options)
}
return query, headers
}

View file

@ -0,0 +1,462 @@
package storage
import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"sync"
)
const fourMB = uint64(4194304)
const oneTB = uint64(1099511627776)
// File represents a file on a share.
type File struct {
fsc *FileServiceClient
Metadata map[string]string
Name string `xml:"Name"`
parent *Directory
Properties FileProperties `xml:"Properties"`
share *Share
FileCopyProperties FileCopyState
mutex *sync.Mutex
}
// FileProperties contains various properties of a file.
type FileProperties struct {
CacheControl string `header:"x-ms-cache-control"`
Disposition string `header:"x-ms-content-disposition"`
Encoding string `header:"x-ms-content-encoding"`
Etag string
Language string `header:"x-ms-content-language"`
LastModified string
Length uint64 `xml:"Content-Length" header:"x-ms-content-length"`
MD5 string `header:"x-ms-content-md5"`
Type string `header:"x-ms-content-type"`
}
// FileCopyState contains various properties of a file copy operation.
type FileCopyState struct {
CompletionTime string
ID string `header:"x-ms-copy-id"`
Progress string
Source string
Status string `header:"x-ms-copy-status"`
StatusDesc string
}
// FileStream contains file data returned from a call to GetFile.
type FileStream struct {
Body io.ReadCloser
ContentMD5 string
}
// FileRequestOptions will be passed to misc file operations.
// Currently just Timeout (in seconds) but could expand.
type FileRequestOptions struct {
Timeout uint // timeout duration in seconds.
}
func prepareOptions(options *FileRequestOptions) url.Values {
params := url.Values{}
if options != nil {
params = addTimeout(params, options.Timeout)
}
return params
}
// FileRanges contains a list of file range information for a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
type FileRanges struct {
ContentLength uint64
LastModified string
ETag string
FileRanges []FileRange `xml:"Range"`
}
// FileRange contains range information for a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
type FileRange struct {
Start uint64 `xml:"Start"`
End uint64 `xml:"End"`
}
func (fr FileRange) String() string {
return fmt.Sprintf("bytes=%d-%d", fr.Start, fr.End)
}
// builds the complete file path for this file object
func (f *File) buildPath() string {
return f.parent.buildPath() + "/" + f.Name
}
// ClearRange releases the specified range of space in a file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
func (f *File) ClearRange(fileRange FileRange, options *FileRequestOptions) error {
var timeout *uint
if options != nil {
timeout = &options.Timeout
}
headers, err := f.modifyRange(nil, fileRange, timeout, nil)
if err != nil {
return err
}
f.updateEtagAndLastModified(headers)
return nil
}
// Create creates a new file or replaces an existing one.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-File
func (f *File) Create(maxSize uint64, options *FileRequestOptions) error {
if maxSize > oneTB {
return fmt.Errorf("max file size is 1TB")
}
params := prepareOptions(options)
headers := headersFromStruct(f.Properties)
headers["x-ms-content-length"] = strconv.FormatUint(maxSize, 10)
headers["x-ms-type"] = "file"
outputHeaders, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, headers), []int{http.StatusCreated})
if err != nil {
return err
}
f.Properties.Length = maxSize
f.updateEtagAndLastModified(outputHeaders)
return nil
}
// CopyFile operation copied a file/blob from the sourceURL to the path provided.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/copy-file
func (f *File) CopyFile(sourceURL string, options *FileRequestOptions) error {
extraHeaders := map[string]string{
"x-ms-type": "file",
"x-ms-copy-source": sourceURL,
}
params := prepareOptions(options)
headers, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, extraHeaders), []int{http.StatusAccepted})
if err != nil {
return err
}
f.updateEtagAndLastModified(headers)
f.FileCopyProperties.ID = headers.Get("X-Ms-Copy-Id")
f.FileCopyProperties.Status = headers.Get("X-Ms-Copy-Status")
return nil
}
// Delete immediately removes this file from the storage account.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
func (f *File) Delete(options *FileRequestOptions) error {
return f.fsc.deleteResource(f.buildPath(), resourceFile, options)
}
// DeleteIfExists removes this file if it exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
func (f *File) DeleteIfExists(options *FileRequestOptions) (bool, error) {
resp, err := f.fsc.deleteResourceNoClose(f.buildPath(), resourceFile, options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
// GetFileOptions includes options for a get file operation
type GetFileOptions struct {
Timeout uint
GetContentMD5 bool
}
// DownloadToStream operation downloads the file.
//
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
func (f *File) DownloadToStream(options *FileRequestOptions) (io.ReadCloser, error) {
params := prepareOptions(options)
resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, nil)
if err != nil {
return nil, err
}
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
readAndCloseBody(resp.body)
return nil, err
}
return resp.body, nil
}
// DownloadRangeToStream operation downloads the specified range of this file with optional MD5 hash.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
func (f *File) DownloadRangeToStream(fileRange FileRange, options *GetFileOptions) (fs FileStream, err error) {
extraHeaders := map[string]string{
"Range": fileRange.String(),
}
params := url.Values{}
if options != nil {
if options.GetContentMD5 {
if isRangeTooBig(fileRange) {
return fs, fmt.Errorf("must specify a range less than or equal to 4MB when getContentMD5 is true")
}
extraHeaders["x-ms-range-get-content-md5"] = "true"
}
params = addTimeout(params, options.Timeout)
}
resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, extraHeaders)
if err != nil {
return fs, err
}
if err = checkRespCode(resp.statusCode, []int{http.StatusOK, http.StatusPartialContent}); err != nil {
readAndCloseBody(resp.body)
return fs, err
}
fs.Body = resp.body
if options != nil && options.GetContentMD5 {
fs.ContentMD5 = resp.headers.Get("Content-MD5")
}
return fs, nil
}
// Exists returns true if this file exists.
func (f *File) Exists() (bool, error) {
exists, headers, err := f.fsc.resourceExists(f.buildPath(), resourceFile)
if exists {
f.updateEtagAndLastModified(headers)
f.updateProperties(headers)
}
return exists, err
}
// FetchAttributes updates metadata and properties for this file.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-properties
func (f *File) FetchAttributes(options *FileRequestOptions) error {
params := prepareOptions(options)
headers, err := f.fsc.getResourceHeaders(f.buildPath(), compNone, resourceFile, params, http.MethodHead)
if err != nil {
return err
}
f.updateEtagAndLastModified(headers)
f.updateProperties(headers)
f.Metadata = getMetadataFromHeaders(headers)
return nil
}
// returns true if the range is larger than 4MB
func isRangeTooBig(fileRange FileRange) bool {
if fileRange.End-fileRange.Start > fourMB {
return true
}
return false
}
// ListRangesOptions includes options for a list file ranges operation
type ListRangesOptions struct {
Timeout uint
ListRange *FileRange
}
// ListRanges returns the list of valid ranges for this file.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
func (f *File) ListRanges(options *ListRangesOptions) (*FileRanges, error) {
params := url.Values{"comp": {"rangelist"}}
// add optional range to list
var headers map[string]string
if options != nil {
params = addTimeout(params, options.Timeout)
if options.ListRange != nil {
headers = make(map[string]string)
headers["Range"] = options.ListRange.String()
}
}
resp, err := f.fsc.listContent(f.buildPath(), params, headers)
if err != nil {
return nil, err
}
defer resp.body.Close()
var cl uint64
cl, err = strconv.ParseUint(resp.headers.Get("x-ms-content-length"), 10, 64)
if err != nil {
ioutil.ReadAll(resp.body)
return nil, err
}
var out FileRanges
out.ContentLength = cl
out.ETag = resp.headers.Get("ETag")
out.LastModified = resp.headers.Get("Last-Modified")
err = xmlUnmarshal(resp.body, &out)
return &out, err
}
// modifies a range of bytes in this file
func (f *File) modifyRange(bytes io.Reader, fileRange FileRange, timeout *uint, contentMD5 *string) (http.Header, error) {
if err := f.fsc.checkForStorageEmulator(); err != nil {
return nil, err
}
if fileRange.End < fileRange.Start {
return nil, errors.New("the value for rangeEnd must be greater than or equal to rangeStart")
}
if bytes != nil && isRangeTooBig(fileRange) {
return nil, errors.New("range cannot exceed 4MB in size")
}
params := url.Values{"comp": {"range"}}
if timeout != nil {
params = addTimeout(params, *timeout)
}
uri := f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), params)
// default to clear
write := "clear"
cl := uint64(0)
// if bytes is not nil then this is an update operation
if bytes != nil {
write = "update"
cl = (fileRange.End - fileRange.Start) + 1
}
extraHeaders := map[string]string{
"Content-Length": strconv.FormatUint(cl, 10),
"Range": fileRange.String(),
"x-ms-write": write,
}
if contentMD5 != nil {
extraHeaders["Content-MD5"] = *contentMD5
}
headers := mergeHeaders(f.fsc.client.getStandardHeaders(), extraHeaders)
resp, err := f.fsc.client.exec(http.MethodPut, uri, headers, bytes, f.fsc.auth)
if err != nil {
return nil, err
}
defer readAndCloseBody(resp.body)
return resp.headers, checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// SetMetadata replaces the metadata for this file.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetFileMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Metadata
func (f *File) SetMetadata(options *FileRequestOptions) error {
headers, err := f.fsc.setResourceHeaders(f.buildPath(), compMetadata, resourceFile, mergeMDIntoExtraHeaders(f.Metadata, nil), options)
if err != nil {
return err
}
f.updateEtagAndLastModified(headers)
return nil
}
// SetProperties sets system properties on this file.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by SetFileProperties. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Properties
func (f *File) SetProperties(options *FileRequestOptions) error {
headers, err := f.fsc.setResourceHeaders(f.buildPath(), compProperties, resourceFile, headersFromStruct(f.Properties), options)
if err != nil {
return err
}
f.updateEtagAndLastModified(headers)
return nil
}
// updates Etag and last modified date
func (f *File) updateEtagAndLastModified(headers http.Header) {
f.Properties.Etag = headers.Get("Etag")
f.Properties.LastModified = headers.Get("Last-Modified")
}
// updates file properties from the specified HTTP header
func (f *File) updateProperties(header http.Header) {
size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64)
if err == nil {
f.Properties.Length = size
}
f.updateEtagAndLastModified(header)
f.Properties.CacheControl = header.Get("Cache-Control")
f.Properties.Disposition = header.Get("Content-Disposition")
f.Properties.Encoding = header.Get("Content-Encoding")
f.Properties.Language = header.Get("Content-Language")
f.Properties.MD5 = header.Get("Content-MD5")
f.Properties.Type = header.Get("Content-Type")
}
// URL gets the canonical URL to this file.
// This method does not create a publicly accessible URL if the file
// is private and this method does not check if the file exists.
func (f *File) URL() string {
return f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), nil)
}
// WriteRangeOptions includes options for a write file range operation
type WriteRangeOptions struct {
Timeout uint
ContentMD5 string
}
// WriteRange writes a range of bytes to this file with an optional MD5 hash of the content (inside
// options parameter). Note that the length of bytes must match (rangeEnd - rangeStart) + 1 with
// a maximum size of 4MB.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
func (f *File) WriteRange(bytes io.Reader, fileRange FileRange, options *WriteRangeOptions) error {
if bytes == nil {
return errors.New("bytes cannot be nil")
}
var timeout *uint
var md5 *string
if options != nil {
timeout = &options.Timeout
md5 = &options.ContentMD5
}
headers, err := f.modifyRange(bytes, fileRange, timeout, md5)
if err != nil {
return err
}
// it's perfectly legal for multiple go routines to call WriteRange
// on the same *File (e.g. concurrently writing non-overlapping ranges)
// so we must take the file mutex before updating our properties.
f.mutex.Lock()
f.updateEtagAndLastModified(headers)
f.mutex.Unlock()
return nil
}

View file

@ -0,0 +1,324 @@
package storage
import (
"encoding/xml"
"fmt"
"net/http"
"net/url"
"strconv"
)
// FileServiceClient contains operations for Microsoft Azure File Service.
type FileServiceClient struct {
client Client
auth authentication
}
// ListSharesParameters defines the set of customizable parameters to make a
// List Shares call.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Shares
type ListSharesParameters struct {
Prefix string
Marker string
Include string
MaxResults uint
Timeout uint
}
// ShareListResponse contains the response fields from
// ListShares call.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Shares
type ShareListResponse struct {
XMLName xml.Name `xml:"EnumerationResults"`
Xmlns string `xml:"xmlns,attr"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker"`
MaxResults int64 `xml:"MaxResults"`
Shares []Share `xml:"Shares>Share"`
}
type compType string
const (
compNone compType = ""
compList compType = "list"
compMetadata compType = "metadata"
compProperties compType = "properties"
compRangeList compType = "rangelist"
)
func (ct compType) String() string {
return string(ct)
}
type resourceType string
const (
resourceDirectory resourceType = "directory"
resourceFile resourceType = ""
resourceShare resourceType = "share"
)
func (rt resourceType) String() string {
return string(rt)
}
func (p ListSharesParameters) getParameters() url.Values {
out := url.Values{}
if p.Prefix != "" {
out.Set("prefix", p.Prefix)
}
if p.Marker != "" {
out.Set("marker", p.Marker)
}
if p.Include != "" {
out.Set("include", p.Include)
}
if p.MaxResults != 0 {
out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
}
if p.Timeout != 0 {
out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
}
return out
}
func (p ListDirsAndFilesParameters) getParameters() url.Values {
out := url.Values{}
if p.Prefix != "" {
out.Set("prefix", p.Prefix)
}
if p.Marker != "" {
out.Set("marker", p.Marker)
}
if p.MaxResults != 0 {
out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
}
out = addTimeout(out, p.Timeout)
return out
}
// returns url.Values for the specified types
func getURLInitValues(comp compType, res resourceType) url.Values {
values := url.Values{}
if comp != compNone {
values.Set("comp", comp.String())
}
if res != resourceFile {
values.Set("restype", res.String())
}
return values
}
// GetShareReference returns a Share object for the specified share name.
func (f *FileServiceClient) GetShareReference(name string) *Share {
return &Share{
fsc: f,
Name: name,
Properties: ShareProperties{
Quota: -1,
},
}
}
// ListShares returns the list of shares in a storage account along with
// pagination token and other response details.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/list-shares
func (f FileServiceClient) ListShares(params ListSharesParameters) (*ShareListResponse, error) {
q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}})
var out ShareListResponse
resp, err := f.listContent("", q, nil)
if err != nil {
return nil, err
}
defer resp.body.Close()
err = xmlUnmarshal(resp.body, &out)
// assign our client to the newly created Share objects
for i := range out.Shares {
out.Shares[i].fsc = &f
}
return &out, err
}
// GetServiceProperties gets the properties of your storage account's file service.
// File service does not support logging
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-service-properties
func (f *FileServiceClient) GetServiceProperties() (*ServiceProperties, error) {
return f.client.getServiceProperties(fileServiceName, f.auth)
}
// SetServiceProperties sets the properties of your storage account's file service.
// File service does not support logging
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-file-service-properties
func (f *FileServiceClient) SetServiceProperties(props ServiceProperties) error {
return f.client.setServiceProperties(props, fileServiceName, f.auth)
}
// retrieves directory or share content
func (f FileServiceClient) listContent(path string, params url.Values, extraHeaders map[string]string) (*storageResponse, error) {
if err := f.checkForStorageEmulator(); err != nil {
return nil, err
}
uri := f.client.getEndpoint(fileServiceName, path, params)
extraHeaders = f.client.protectUserAgent(extraHeaders)
headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
resp, err := f.client.exec(http.MethodGet, uri, headers, nil, f.auth)
if err != nil {
return nil, err
}
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
readAndCloseBody(resp.body)
return nil, err
}
return resp, nil
}
// returns true if the specified resource exists
func (f FileServiceClient) resourceExists(path string, res resourceType) (bool, http.Header, error) {
if err := f.checkForStorageEmulator(); err != nil {
return false, nil, err
}
uri := f.client.getEndpoint(fileServiceName, path, getURLInitValues(compNone, res))
headers := f.client.getStandardHeaders()
resp, err := f.client.exec(http.MethodHead, uri, headers, nil, f.auth)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusOK, resp.headers, nil
}
}
return false, nil, err
}
// creates a resource depending on the specified resource type
func (f FileServiceClient) createResource(path string, res resourceType, urlParams url.Values, extraHeaders map[string]string, expectedResponseCodes []int) (http.Header, error) {
resp, err := f.createResourceNoClose(path, res, urlParams, extraHeaders)
if err != nil {
return nil, err
}
defer readAndCloseBody(resp.body)
return resp.headers, checkRespCode(resp.statusCode, expectedResponseCodes)
}
// creates a resource depending on the specified resource type, doesn't close the response body
func (f FileServiceClient) createResourceNoClose(path string, res resourceType, urlParams url.Values, extraHeaders map[string]string) (*storageResponse, error) {
if err := f.checkForStorageEmulator(); err != nil {
return nil, err
}
values := getURLInitValues(compNone, res)
combinedParams := mergeParams(values, urlParams)
uri := f.client.getEndpoint(fileServiceName, path, combinedParams)
extraHeaders = f.client.protectUserAgent(extraHeaders)
headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
return f.client.exec(http.MethodPut, uri, headers, nil, f.auth)
}
// returns HTTP header data for the specified directory or share
func (f FileServiceClient) getResourceHeaders(path string, comp compType, res resourceType, params url.Values, verb string) (http.Header, error) {
resp, err := f.getResourceNoClose(path, comp, res, params, verb, nil)
if err != nil {
return nil, err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
return resp.headers, nil
}
// gets the specified resource, doesn't close the response body
func (f FileServiceClient) getResourceNoClose(path string, comp compType, res resourceType, params url.Values, verb string, extraHeaders map[string]string) (*storageResponse, error) {
if err := f.checkForStorageEmulator(); err != nil {
return nil, err
}
params = mergeParams(params, getURLInitValues(comp, res))
uri := f.client.getEndpoint(fileServiceName, path, params)
headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
return f.client.exec(verb, uri, headers, nil, f.auth)
}
// deletes the resource and returns the response
func (f FileServiceClient) deleteResource(path string, res resourceType, options *FileRequestOptions) error {
resp, err := f.deleteResourceNoClose(path, res, options)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusAccepted})
}
// deletes the resource and returns the response, doesn't close the response body
func (f FileServiceClient) deleteResourceNoClose(path string, res resourceType, options *FileRequestOptions) (*storageResponse, error) {
if err := f.checkForStorageEmulator(); err != nil {
return nil, err
}
values := mergeParams(getURLInitValues(compNone, res), prepareOptions(options))
uri := f.client.getEndpoint(fileServiceName, path, values)
return f.client.exec(http.MethodDelete, uri, f.client.getStandardHeaders(), nil, f.auth)
}
// merges metadata into extraHeaders and returns extraHeaders
func mergeMDIntoExtraHeaders(metadata, extraHeaders map[string]string) map[string]string {
if metadata == nil && extraHeaders == nil {
return nil
}
if extraHeaders == nil {
extraHeaders = make(map[string]string)
}
for k, v := range metadata {
extraHeaders[userDefinedMetadataHeaderPrefix+k] = v
}
return extraHeaders
}
// sets extra header data for the specified resource
func (f FileServiceClient) setResourceHeaders(path string, comp compType, res resourceType, extraHeaders map[string]string, options *FileRequestOptions) (http.Header, error) {
if err := f.checkForStorageEmulator(); err != nil {
return nil, err
}
params := mergeParams(getURLInitValues(comp, res), prepareOptions(options))
uri := f.client.getEndpoint(fileServiceName, path, params)
extraHeaders = f.client.protectUserAgent(extraHeaders)
headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
resp, err := f.client.exec(http.MethodPut, uri, headers, nil, f.auth)
if err != nil {
return nil, err
}
defer readAndCloseBody(resp.body)
return resp.headers, checkRespCode(resp.statusCode, []int{http.StatusOK})
}
//checkForStorageEmulator determines if the client is setup for use with
//Azure Storage Emulator, and returns a relevant error
func (f FileServiceClient) checkForStorageEmulator() error {
if f.client.accountName == StorageEmulatorAccountName {
return fmt.Errorf("Error: File service is not currently supported by Azure Storage Emulator")
}
return nil
}

View file

@ -0,0 +1,187 @@
package storage
import (
"errors"
"net/http"
"net/url"
"strconv"
"time"
)
// lease constants.
const (
leaseHeaderPrefix = "x-ms-lease-"
headerLeaseID = "x-ms-lease-id"
leaseAction = "x-ms-lease-action"
leaseBreakPeriod = "x-ms-lease-break-period"
leaseDuration = "x-ms-lease-duration"
leaseProposedID = "x-ms-proposed-lease-id"
leaseTime = "x-ms-lease-time"
acquireLease = "acquire"
renewLease = "renew"
changeLease = "change"
releaseLease = "release"
breakLease = "break"
)
// leasePut is common PUT code for the various acquire/release/break etc functions.
func (b *Blob) leaseCommonPut(headers map[string]string, expectedStatus int, options *LeaseOptions) (http.Header, error) {
params := url.Values{"comp": {"lease"}}
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return nil, err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{expectedStatus}); err != nil {
return nil, err
}
return resp.headers, nil
}
// LeaseOptions includes options for all operations regarding leasing blobs
type LeaseOptions struct {
Timeout uint
Origin string `header:"Origin"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
RequestID string `header:"x-ms-client-request-id"`
}
// AcquireLease creates a lease for a blob
// returns leaseID acquired
// In API Versions starting on 2012-02-12, the minimum leaseTimeInSeconds is 15, the maximum
// non-infinite leaseTimeInSeconds is 60. To specify an infinite lease, provide the value -1.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
func (b *Blob) AcquireLease(leaseTimeInSeconds int, proposedLeaseID string, options *LeaseOptions) (returnedLeaseID string, err error) {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = acquireLease
if leaseTimeInSeconds == -1 {
// Do nothing, but don't trigger the following clauses.
} else if leaseTimeInSeconds > 60 || b.Container.bsc.client.apiVersion < "2012-02-12" {
leaseTimeInSeconds = 60
} else if leaseTimeInSeconds < 15 {
leaseTimeInSeconds = 15
}
headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds)
if proposedLeaseID != "" {
headers[leaseProposedID] = proposedLeaseID
}
respHeaders, err := b.leaseCommonPut(headers, http.StatusCreated, options)
if err != nil {
return "", err
}
returnedLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID))
if returnedLeaseID != "" {
return returnedLeaseID, nil
}
return "", errors.New("LeaseID not returned")
}
// BreakLease breaks the lease for a blob
// Returns the timeout remaining in the lease in seconds
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
func (b *Blob) BreakLease(options *LeaseOptions) (breakTimeout int, err error) {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = breakLease
return b.breakLeaseCommon(headers, options)
}
// BreakLeaseWithBreakPeriod breaks the lease for a blob
// breakPeriodInSeconds is used to determine how long until new lease can be created.
// Returns the timeout remaining in the lease in seconds
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
func (b *Blob) BreakLeaseWithBreakPeriod(breakPeriodInSeconds int, options *LeaseOptions) (breakTimeout int, err error) {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = breakLease
headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds)
return b.breakLeaseCommon(headers, options)
}
// breakLeaseCommon is common code for both version of BreakLease (with and without break period)
func (b *Blob) breakLeaseCommon(headers map[string]string, options *LeaseOptions) (breakTimeout int, err error) {
respHeaders, err := b.leaseCommonPut(headers, http.StatusAccepted, options)
if err != nil {
return 0, err
}
breakTimeoutStr := respHeaders.Get(http.CanonicalHeaderKey(leaseTime))
if breakTimeoutStr != "" {
breakTimeout, err = strconv.Atoi(breakTimeoutStr)
if err != nil {
return 0, err
}
}
return breakTimeout, nil
}
// ChangeLease changes a lease ID for a blob
// Returns the new LeaseID acquired
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
func (b *Blob) ChangeLease(currentLeaseID string, proposedLeaseID string, options *LeaseOptions) (newLeaseID string, err error) {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = changeLease
headers[headerLeaseID] = currentLeaseID
headers[leaseProposedID] = proposedLeaseID
respHeaders, err := b.leaseCommonPut(headers, http.StatusOK, options)
if err != nil {
return "", err
}
newLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID))
if newLeaseID != "" {
return newLeaseID, nil
}
return "", errors.New("LeaseID not returned")
}
// ReleaseLease releases the lease for a blob
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
func (b *Blob) ReleaseLease(currentLeaseID string, options *LeaseOptions) error {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = releaseLease
headers[headerLeaseID] = currentLeaseID
_, err := b.leaseCommonPut(headers, http.StatusOK, options)
if err != nil {
return err
}
return nil
}
// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
func (b *Blob) RenewLease(currentLeaseID string, options *LeaseOptions) error {
headers := b.Container.bsc.client.getStandardHeaders()
headers[leaseAction] = renewLease
headers[headerLeaseID] = currentLeaseID
_, err := b.leaseCommonPut(headers, http.StatusOK, options)
if err != nil {
return err
}
return nil
}

View file

@ -0,0 +1,153 @@
package storage
import (
"encoding/xml"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
)
// Message represents an Azure message.
type Message struct {
Queue *Queue
Text string `xml:"MessageText"`
ID string `xml:"MessageId"`
Insertion TimeRFC1123 `xml:"InsertionTime"`
Expiration TimeRFC1123 `xml:"ExpirationTime"`
PopReceipt string `xml:"PopReceipt"`
NextVisible TimeRFC1123 `xml:"TimeNextVisible"`
DequeueCount int `xml:"DequeueCount"`
}
func (m *Message) buildPath() string {
return fmt.Sprintf("%s/%s", m.Queue.buildPathMessages(), m.ID)
}
// PutMessageOptions is the set of options can be specified for Put Messsage
// operation. A zero struct does not use any preferences for the request.
type PutMessageOptions struct {
Timeout uint
VisibilityTimeout int
MessageTTL int
RequestID string `header:"x-ms-client-request-id"`
}
// Put operation adds a new message to the back of the message queue.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Message
func (m *Message) Put(options *PutMessageOptions) error {
query := url.Values{}
headers := m.Queue.qsc.client.getStandardHeaders()
req := putMessageRequest{MessageText: m.Text}
body, nn, err := xmlMarshal(req)
if err != nil {
return err
}
headers["Content-Length"] = strconv.Itoa(nn)
if options != nil {
if options.VisibilityTimeout != 0 {
query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
}
if options.MessageTTL != 0 {
query.Set("messagettl", strconv.Itoa(options.MessageTTL))
}
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.Queue.buildPathMessages(), query)
resp, err := m.Queue.qsc.client.exec(http.MethodPost, uri, headers, body, m.Queue.qsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
err = xmlUnmarshal(resp.body, m)
if err != nil {
return err
}
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// UpdateMessageOptions is the set of options can be specified for Update Messsage
// operation. A zero struct does not use any preferences for the request.
type UpdateMessageOptions struct {
Timeout uint
VisibilityTimeout int
RequestID string `header:"x-ms-client-request-id"`
}
// Update operation updates the specified message.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Update-Message
func (m *Message) Update(options *UpdateMessageOptions) error {
query := url.Values{}
if m.PopReceipt != "" {
query.Set("popreceipt", m.PopReceipt)
}
headers := m.Queue.qsc.client.getStandardHeaders()
req := putMessageRequest{MessageText: m.Text}
body, nn, err := xmlMarshal(req)
if err != nil {
return err
}
headers["Content-Length"] = strconv.Itoa(nn)
if options != nil {
if options.VisibilityTimeout != 0 {
query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
}
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.buildPath(), query)
resp, err := m.Queue.qsc.client.exec(http.MethodPut, uri, headers, body, m.Queue.qsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
m.PopReceipt = resp.headers.Get("x-ms-popreceipt")
nextTimeStr := resp.headers.Get("x-ms-time-next-visible")
if nextTimeStr != "" {
nextTime, err := time.Parse(time.RFC1123, nextTimeStr)
if err != nil {
return err
}
m.NextVisible = TimeRFC1123(nextTime)
}
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// Delete operation deletes the specified message.
//
// See https://msdn.microsoft.com/en-us/library/azure/dd179347.aspx
func (m *Message) Delete(options *QueueServiceOptions) error {
params := url.Values{"popreceipt": {m.PopReceipt}}
headers := m.Queue.qsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.buildPath(), params)
resp, err := m.Queue.qsc.client.exec(http.MethodDelete, uri, headers, nil, m.Queue.qsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
type putMessageRequest struct {
XMLName xml.Name `xml:"QueueMessage"`
MessageText string `xml:"MessageText"`
}

View file

@ -0,0 +1,33 @@
package storage
// MetadataLevel determines if operations should return a paylod,
// and it level of detail.
type MetadataLevel string
// This consts are meant to help with Odata supported operations
const (
OdataTypeSuffix = "@odata.type"
// Types
OdataBinary = "Edm.Binary"
OdataDateTime = "Edm.DateTime"
OdataGUID = "Edm.Guid"
OdataInt64 = "Edm.Int64"
// Query options
OdataFilter = "$filter"
OdataOrderBy = "$orderby"
OdataTop = "$top"
OdataSkip = "$skip"
OdataCount = "$count"
OdataExpand = "$expand"
OdataSelect = "$select"
OdataSearch = "$search"
EmptyPayload MetadataLevel = ""
NoMetadata MetadataLevel = "application/json;odata=nometadata"
MinimalMetadata MetadataLevel = "application/json;odata=minimalmetadata"
FullMetadata MetadataLevel = "application/json;odata=fullmetadata"
)

View file

@ -0,0 +1,191 @@
package storage
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
)
// GetPageRangesResponse contains the response fields from
// Get Page Ranges call.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx
type GetPageRangesResponse struct {
XMLName xml.Name `xml:"PageList"`
PageList []PageRange `xml:"PageRange"`
}
// PageRange contains information about a page of a page blob from
// Get Pages Range call.
//
// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx
type PageRange struct {
Start int64 `xml:"Start"`
End int64 `xml:"End"`
}
var (
errBlobCopyAborted = errors.New("storage: blob copy is aborted")
errBlobCopyIDMismatch = errors.New("storage: blob copy id is a mismatch")
)
// PutPageOptions includes the options for a put page operation
type PutPageOptions struct {
Timeout uint
LeaseID string `header:"x-ms-lease-id"`
IfSequenceNumberLessThanOrEqualTo *int `header:"x-ms-if-sequence-number-le"`
IfSequenceNumberLessThan *int `header:"x-ms-if-sequence-number-lt"`
IfSequenceNumberEqualTo *int `header:"x-ms-if-sequence-number-eq"`
IfModifiedSince *time.Time `header:"If-Modified-Since"`
IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
IfMatch string `header:"If-Match"`
IfNoneMatch string `header:"If-None-Match"`
RequestID string `header:"x-ms-client-request-id"`
}
// WriteRange writes a range of pages to a page blob.
// Ranges must be aligned with 512-byte boundaries and chunk must be of size
// multiplies by 512.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Page
func (b *Blob) WriteRange(blobRange BlobRange, bytes io.Reader, options *PutPageOptions) error {
if bytes == nil {
return errors.New("bytes cannot be nil")
}
return b.modifyRange(blobRange, bytes, options)
}
// ClearRange clears the given range in a page blob.
// Ranges must be aligned with 512-byte boundaries and chunk must be of size
// multiplies by 512.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Page
func (b *Blob) ClearRange(blobRange BlobRange, options *PutPageOptions) error {
return b.modifyRange(blobRange, nil, options)
}
func (b *Blob) modifyRange(blobRange BlobRange, bytes io.Reader, options *PutPageOptions) error {
if blobRange.End < blobRange.Start {
return errors.New("the value for rangeEnd must be greater than or equal to rangeStart")
}
if blobRange.Start%512 != 0 {
return errors.New("the value for rangeStart must be a modulus of 512")
}
if blobRange.End%512 != 511 {
return errors.New("the value for rangeEnd must be a modulus of 511")
}
params := url.Values{"comp": {"page"}}
// default to clear
write := "clear"
var cl uint64
// if bytes is not nil then this is an update operation
if bytes != nil {
write = "update"
cl = (blobRange.End - blobRange.Start) + 1
}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypePage)
headers["x-ms-page-write"] = write
headers["x-ms-range"] = blobRange.String()
headers["Content-Length"] = fmt.Sprintf("%v", cl)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, bytes, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// GetPageRangesOptions includes the options for a get page ranges operation
type GetPageRangesOptions struct {
Timeout uint
Snapshot *time.Time
PreviousSnapshot *time.Time
Range *BlobRange
LeaseID string `header:"x-ms-lease-id"`
RequestID string `header:"x-ms-client-request-id"`
}
// GetPageRanges returns the list of valid page ranges for a page blob.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Page-Ranges
func (b *Blob) GetPageRanges(options *GetPageRangesOptions) (GetPageRangesResponse, error) {
params := url.Values{"comp": {"pagelist"}}
headers := b.Container.bsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
params = addSnapshot(params, options.Snapshot)
if options.PreviousSnapshot != nil {
params.Add("prevsnapshot", timeRfc1123Formatted(*options.PreviousSnapshot))
}
if options.Range != nil {
headers["Range"] = options.Range.String()
}
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
var out GetPageRangesResponse
resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return out, err
}
defer resp.body.Close()
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return out, err
}
err = xmlUnmarshal(resp.body, &out)
return out, err
}
// PutPageBlob initializes an empty page blob with specified name and maximum
// size in bytes (size must be aligned to a 512-byte boundary). A page blob must
// be created using this method before writing pages.
//
// See CreateBlockBlobFromReader for more info on creating blobs.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
func (b *Blob) PutPageBlob(options *PutBlobOptions) error {
if b.Properties.ContentLength%512 != 0 {
return errors.New("Content length must be aligned to a 512-byte boundary")
}
params := url.Values{}
headers := b.Container.bsc.client.getStandardHeaders()
headers["x-ms-blob-type"] = string(BlobTypePage)
headers["x-ms-blob-content-length"] = fmt.Sprintf("%v", b.Properties.ContentLength)
headers["x-ms-blob-sequence-number"] = fmt.Sprintf("%v", b.Properties.SequenceNumber)
headers = mergeHeaders(headers, headersFromStruct(b.Properties))
headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}

View file

@ -0,0 +1,427 @@
package storage
import (
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
)
const (
// casing is per Golang's http.Header canonicalizing the header names.
approximateMessagesCountHeader = "X-Ms-Approximate-Messages-Count"
)
// QueueAccessPolicy represents each access policy in the queue ACL.
type QueueAccessPolicy struct {
ID string
StartTime time.Time
ExpiryTime time.Time
CanRead bool
CanAdd bool
CanUpdate bool
CanProcess bool
}
// QueuePermissions represents the queue ACLs.
type QueuePermissions struct {
AccessPolicies []QueueAccessPolicy
}
// SetQueuePermissionOptions includes options for a set queue permissions operation
type SetQueuePermissionOptions struct {
Timeout uint
RequestID string `header:"x-ms-client-request-id"`
}
// Queue represents an Azure queue.
type Queue struct {
qsc *QueueServiceClient
Name string
Metadata map[string]string
AproxMessageCount uint64
}
func (q *Queue) buildPath() string {
return fmt.Sprintf("/%s", q.Name)
}
func (q *Queue) buildPathMessages() string {
return fmt.Sprintf("%s/messages", q.buildPath())
}
// QueueServiceOptions includes options for some queue service operations
type QueueServiceOptions struct {
Timeout uint
RequestID string `header:"x-ms-client-request-id"`
}
// Create operation creates a queue under the given account.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Queue4
func (q *Queue) Create(options *QueueServiceOptions) error {
params := url.Values{}
headers := q.qsc.client.getStandardHeaders()
headers = q.qsc.client.addMetadataToHeaders(headers, q.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, nil, q.qsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusCreated})
}
// Delete operation permanently deletes the specified queue.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Queue3
func (q *Queue) Delete(options *QueueServiceOptions) error {
params := url.Values{}
headers := q.qsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
resp, err := q.qsc.client.exec(http.MethodDelete, uri, headers, nil, q.qsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// Exists returns true if a queue with given name exists.
func (q *Queue) Exists() (bool, error) {
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), url.Values{"comp": {"metadata"}})
resp, err := q.qsc.client.exec(http.MethodGet, uri, q.qsc.client.getStandardHeaders(), nil, q.qsc.auth)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusOK, nil
}
}
return false, err
}
// SetMetadata operation sets user-defined metadata on the specified queue.
// Metadata is associated with the queue as name-value pairs.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Queue-Metadata
func (q *Queue) SetMetadata(options *QueueServiceOptions) error {
params := url.Values{"comp": {"metadata"}}
headers := q.qsc.client.getStandardHeaders()
headers = q.qsc.client.addMetadataToHeaders(headers, q.Metadata)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, nil, q.qsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// GetMetadata operation retrieves user-defined metadata and queue
// properties on the specified queue. Metadata is associated with
// the queue as name-values pairs.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Queue-Metadata
//
// Because the way Golang's http client (and http.Header in particular)
// canonicalize header names, the returned metadata names would always
// be all lower case.
func (q *Queue) GetMetadata(options *QueueServiceOptions) error {
params := url.Values{"comp": {"metadata"}}
headers := q.qsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), url.Values{"comp": {"metadata"}})
resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return err
}
aproxMessagesStr := resp.headers.Get(http.CanonicalHeaderKey(approximateMessagesCountHeader))
if aproxMessagesStr != "" {
aproxMessages, err := strconv.ParseUint(aproxMessagesStr, 10, 64)
if err != nil {
return err
}
q.AproxMessageCount = aproxMessages
}
q.Metadata = getMetadataFromHeaders(resp.headers)
return nil
}
// GetMessageReference returns a message object with the specified text.
func (q *Queue) GetMessageReference(text string) *Message {
return &Message{
Queue: q,
Text: text,
}
}
// GetMessagesOptions is the set of options can be specified for Get
// Messsages operation. A zero struct does not use any preferences for the
// request.
type GetMessagesOptions struct {
Timeout uint
NumOfMessages int
VisibilityTimeout int
RequestID string `header:"x-ms-client-request-id"`
}
type messages struct {
XMLName xml.Name `xml:"QueueMessagesList"`
Messages []Message `xml:"QueueMessage"`
}
// GetMessages operation retrieves one or more messages from the front of the
// queue.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Messages
func (q *Queue) GetMessages(options *GetMessagesOptions) ([]Message, error) {
query := url.Values{}
headers := q.qsc.client.getStandardHeaders()
if options != nil {
if options.NumOfMessages != 0 {
query.Set("numofmessages", strconv.Itoa(options.NumOfMessages))
}
if options.VisibilityTimeout != 0 {
query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
}
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), query)
resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
if err != nil {
return []Message{}, err
}
defer readAndCloseBody(resp.body)
var out messages
err = xmlUnmarshal(resp.body, &out)
if err != nil {
return []Message{}, err
}
for i := range out.Messages {
out.Messages[i].Queue = q
}
return out.Messages, err
}
// PeekMessagesOptions is the set of options can be specified for Peek
// Messsage operation. A zero struct does not use any preferences for the
// request.
type PeekMessagesOptions struct {
Timeout uint
NumOfMessages int
RequestID string `header:"x-ms-client-request-id"`
}
// PeekMessages retrieves one or more messages from the front of the queue, but
// does not alter the visibility of the message.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Peek-Messages
func (q *Queue) PeekMessages(options *PeekMessagesOptions) ([]Message, error) {
query := url.Values{"peekonly": {"true"}} // Required for peek operation
headers := q.qsc.client.getStandardHeaders()
if options != nil {
if options.NumOfMessages != 0 {
query.Set("numofmessages", strconv.Itoa(options.NumOfMessages))
}
query = addTimeout(query, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), query)
resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
if err != nil {
return []Message{}, err
}
defer readAndCloseBody(resp.body)
var out messages
err = xmlUnmarshal(resp.body, &out)
if err != nil {
return []Message{}, err
}
for i := range out.Messages {
out.Messages[i].Queue = q
}
return out.Messages, err
}
// ClearMessages operation deletes all messages from the specified queue.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Clear-Messages
func (q *Queue) ClearMessages(options *QueueServiceOptions) error {
params := url.Values{}
headers := q.qsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), params)
resp, err := q.qsc.client.exec(http.MethodDelete, uri, headers, nil, q.qsc.auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusNoContent})
}
// SetPermissions sets up queue permissions
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-queue-acl
func (q *Queue) SetPermissions(permissions QueuePermissions, options *SetQueuePermissionOptions) error {
body, length, err := generateQueueACLpayload(permissions.AccessPolicies)
if err != nil {
return err
}
params := url.Values{
"comp": {"acl"},
}
headers := q.qsc.client.getStandardHeaders()
headers["Content-Length"] = strconv.Itoa(length)
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, body, q.qsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return errors.New("Unable to set permissions")
}
return nil
}
func generateQueueACLpayload(policies []QueueAccessPolicy) (io.Reader, int, error) {
sil := SignedIdentifiers{
SignedIdentifiers: []SignedIdentifier{},
}
for _, qapd := range policies {
permission := qapd.generateQueuePermissions()
signedIdentifier := convertAccessPolicyToXMLStructs(qapd.ID, qapd.StartTime, qapd.ExpiryTime, permission)
sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
}
return xmlMarshal(sil)
}
func (qapd *QueueAccessPolicy) generateQueuePermissions() (permissions string) {
// generate the permissions string (raup).
// still want the end user API to have bool flags.
permissions = ""
if qapd.CanRead {
permissions += "r"
}
if qapd.CanAdd {
permissions += "a"
}
if qapd.CanUpdate {
permissions += "u"
}
if qapd.CanProcess {
permissions += "p"
}
return permissions
}
// GetQueuePermissionOptions includes options for a get queue permissions operation
type GetQueuePermissionOptions struct {
Timeout uint
RequestID string `header:"x-ms-client-request-id"`
}
// GetPermissions gets the queue permissions as per https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-queue-acl
// If timeout is 0 then it will not be passed to Azure
func (q *Queue) GetPermissions(options *GetQueuePermissionOptions) (*QueuePermissions, error) {
params := url.Values{
"comp": {"acl"},
}
headers := q.qsc.client.getStandardHeaders()
if options != nil {
params = addTimeout(params, options.Timeout)
headers = mergeHeaders(headers, headersFromStruct(*options))
}
uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
var ap AccessPolicy
err = xmlUnmarshal(resp.body, &ap.SignedIdentifiersList)
if err != nil {
return nil, err
}
return buildQueueAccessPolicy(ap, &resp.headers), nil
}
func buildQueueAccessPolicy(ap AccessPolicy, headers *http.Header) *QueuePermissions {
permissions := QueuePermissions{
AccessPolicies: []QueueAccessPolicy{},
}
for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
qapd := QueueAccessPolicy{
ID: policy.ID,
StartTime: policy.AccessPolicy.StartTime,
ExpiryTime: policy.AccessPolicy.ExpiryTime,
}
qapd.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
qapd.CanAdd = updatePermissions(policy.AccessPolicy.Permission, "a")
qapd.CanUpdate = updatePermissions(policy.AccessPolicy.Permission, "u")
qapd.CanProcess = updatePermissions(policy.AccessPolicy.Permission, "p")
permissions.AccessPolicies = append(permissions.AccessPolicies, qapd)
}
return &permissions
}

View file

@ -0,0 +1,28 @@
package storage
// QueueServiceClient contains operations for Microsoft Azure Queue Storage
// Service.
type QueueServiceClient struct {
client Client
auth authentication
}
// GetServiceProperties gets the properties of your storage account's queue service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-queue-service-properties
func (q *QueueServiceClient) GetServiceProperties() (*ServiceProperties, error) {
return q.client.getServiceProperties(queueServiceName, q.auth)
}
// SetServiceProperties sets the properties of your storage account's queue service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-queue-service-properties
func (q *QueueServiceClient) SetServiceProperties(props ServiceProperties) error {
return q.client.setServiceProperties(props, queueServiceName, q.auth)
}
// GetQueueReference returns a Container object for the specified queue name.
func (q *QueueServiceClient) GetQueueReference(name string) *Queue {
return &Queue{
qsc: q,
Name: name,
}
}

View file

@ -0,0 +1,202 @@
package storage
import (
"fmt"
"net/http"
"net/url"
"strconv"
)
// Share represents an Azure file share.
type Share struct {
fsc *FileServiceClient
Name string `xml:"Name"`
Properties ShareProperties `xml:"Properties"`
Metadata map[string]string
}
// ShareProperties contains various properties of a share.
type ShareProperties struct {
LastModified string `xml:"Last-Modified"`
Etag string `xml:"Etag"`
Quota int `xml:"Quota"`
}
// builds the complete path for this share object.
func (s *Share) buildPath() string {
return fmt.Sprintf("/%s", s.Name)
}
// Create this share under the associated account.
// If a share with the same name already exists, the operation fails.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share
func (s *Share) Create(options *FileRequestOptions) error {
extraheaders := map[string]string{}
if s.Properties.Quota > 0 {
extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
}
params := prepareOptions(options)
headers, err := s.fsc.createResource(s.buildPath(), resourceShare, params, mergeMDIntoExtraHeaders(s.Metadata, extraheaders), []int{http.StatusCreated})
if err != nil {
return err
}
s.updateEtagAndLastModified(headers)
return nil
}
// CreateIfNotExists creates this share under the associated account if
// it does not exist. Returns true if the share is newly created or false if
// the share already exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share
func (s *Share) CreateIfNotExists(options *FileRequestOptions) (bool, error) {
extraheaders := map[string]string{}
if s.Properties.Quota > 0 {
extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
}
params := prepareOptions(options)
resp, err := s.fsc.createResourceNoClose(s.buildPath(), resourceShare, params, extraheaders)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict {
if resp.statusCode == http.StatusCreated {
s.updateEtagAndLastModified(resp.headers)
return true, nil
}
return false, s.FetchAttributes(nil)
}
}
return false, err
}
// Delete marks this share for deletion. The share along with any files
// and directories contained within it are later deleted during garbage
// collection. If the share does not exist the operation fails
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Share
func (s *Share) Delete(options *FileRequestOptions) error {
return s.fsc.deleteResource(s.buildPath(), resourceShare, options)
}
// DeleteIfExists operation marks this share for deletion if it exists.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Share
func (s *Share) DeleteIfExists(options *FileRequestOptions) (bool, error) {
resp, err := s.fsc.deleteResourceNoClose(s.buildPath(), resourceShare, options)
if resp != nil {
defer readAndCloseBody(resp.body)
if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound {
return resp.statusCode == http.StatusAccepted, nil
}
}
return false, err
}
// Exists returns true if this share already exists
// on the storage account, otherwise returns false.
func (s *Share) Exists() (bool, error) {
exists, headers, err := s.fsc.resourceExists(s.buildPath(), resourceShare)
if exists {
s.updateEtagAndLastModified(headers)
s.updateQuota(headers)
}
return exists, err
}
// FetchAttributes retrieves metadata and properties for this share.
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-share-properties
func (s *Share) FetchAttributes(options *FileRequestOptions) error {
params := prepareOptions(options)
headers, err := s.fsc.getResourceHeaders(s.buildPath(), compNone, resourceShare, params, http.MethodHead)
if err != nil {
return err
}
s.updateEtagAndLastModified(headers)
s.updateQuota(headers)
s.Metadata = getMetadataFromHeaders(headers)
return nil
}
// GetRootDirectoryReference returns a Directory object at the root of this share.
func (s *Share) GetRootDirectoryReference() *Directory {
return &Directory{
fsc: s.fsc,
share: s,
}
}
// ServiceClient returns the FileServiceClient associated with this share.
func (s *Share) ServiceClient() *FileServiceClient {
return s.fsc
}
// SetMetadata replaces the metadata for this share.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by GetShareMetadata. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-share-metadata
func (s *Share) SetMetadata(options *FileRequestOptions) error {
headers, err := s.fsc.setResourceHeaders(s.buildPath(), compMetadata, resourceShare, mergeMDIntoExtraHeaders(s.Metadata, nil), options)
if err != nil {
return err
}
s.updateEtagAndLastModified(headers)
return nil
}
// SetProperties sets system properties for this share.
//
// Some keys may be converted to Camel-Case before sending. All keys
// are returned in lower case by SetShareProperties. HTTP header names
// are case-insensitive so case munging should not matter to other
// applications either.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Share-Properties
func (s *Share) SetProperties(options *FileRequestOptions) error {
extraheaders := map[string]string{}
if s.Properties.Quota > 0 {
if s.Properties.Quota > 5120 {
return fmt.Errorf("invalid value %v for quota, valid values are [1, 5120]", s.Properties.Quota)
}
extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
}
headers, err := s.fsc.setResourceHeaders(s.buildPath(), compProperties, resourceShare, extraheaders, options)
if err != nil {
return err
}
s.updateEtagAndLastModified(headers)
return nil
}
// updates Etag and last modified date
func (s *Share) updateEtagAndLastModified(headers http.Header) {
s.Properties.Etag = headers.Get("Etag")
s.Properties.LastModified = headers.Get("Last-Modified")
}
// updates quota value
func (s *Share) updateQuota(headers http.Header) {
quota, err := strconv.Atoi(headers.Get("x-ms-share-quota"))
if err == nil {
s.Properties.Quota = quota
}
}
// URL gets the canonical URL to this share. This method does not create a publicly accessible
// URL if the share is private and this method does not check if the share exists.
func (s *Share) URL() string {
return s.fsc.client.getEndpoint(fileServiceName, s.buildPath(), url.Values{})
}

View file

@ -0,0 +1,47 @@
package storage
import (
"strings"
"time"
)
// AccessPolicyDetailsXML has specifics about an access policy
// annotated with XML details.
type AccessPolicyDetailsXML struct {
StartTime time.Time `xml:"Start"`
ExpiryTime time.Time `xml:"Expiry"`
Permission string `xml:"Permission"`
}
// SignedIdentifier is a wrapper for a specific policy
type SignedIdentifier struct {
ID string `xml:"Id"`
AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"`
}
// SignedIdentifiers part of the response from GetPermissions call.
type SignedIdentifiers struct {
SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"`
}
// AccessPolicy is the response type from the GetPermissions call.
type AccessPolicy struct {
SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"`
}
// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the
// AccessPolicy struct which will get converted to XML.
func convertAccessPolicyToXMLStructs(id string, startTime time.Time, expiryTime time.Time, permissions string) SignedIdentifier {
return SignedIdentifier{
ID: id,
AccessPolicy: AccessPolicyDetailsXML{
StartTime: startTime.UTC().Round(time.Second),
ExpiryTime: expiryTime.UTC().Round(time.Second),
Permission: permissions,
},
}
}
func updatePermissions(permissions, permission string) bool {
return strings.Contains(permissions, permission)
}

View file

@ -0,0 +1,117 @@
package storage
import (
"net/http"
"net/url"
"strconv"
)
// ServiceProperties represents the storage account service properties
type ServiceProperties struct {
Logging *Logging
HourMetrics *Metrics
MinuteMetrics *Metrics
Cors *Cors
}
// Logging represents the Azure Analytics Logging settings
type Logging struct {
Version string
Delete bool
Read bool
Write bool
RetentionPolicy *RetentionPolicy
}
// RetentionPolicy indicates if retention is enabled and for how many days
type RetentionPolicy struct {
Enabled bool
Days *int
}
// Metrics provide request statistics.
type Metrics struct {
Version string
Enabled bool
IncludeAPIs *bool
RetentionPolicy *RetentionPolicy
}
// Cors includes all the CORS rules
type Cors struct {
CorsRule []CorsRule
}
// CorsRule includes all settings for a Cors rule
type CorsRule struct {
AllowedOrigins string
AllowedMethods string
MaxAgeInSeconds int
ExposedHeaders string
AllowedHeaders string
}
func (c Client) getServiceProperties(service string, auth authentication) (*ServiceProperties, error) {
query := url.Values{
"restype": {"service"},
"comp": {"properties"},
}
uri := c.getEndpoint(service, "", query)
headers := c.getStandardHeaders()
resp, err := c.exec(http.MethodGet, uri, headers, nil, auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
var out ServiceProperties
err = xmlUnmarshal(resp.body, &out)
if err != nil {
return nil, err
}
return &out, nil
}
func (c Client) setServiceProperties(props ServiceProperties, service string, auth authentication) error {
query := url.Values{
"restype": {"service"},
"comp": {"properties"},
}
uri := c.getEndpoint(service, "", query)
// Ideally, StorageServiceProperties would be the output struct
// This is to avoid golint stuttering, while generating the correct XML
type StorageServiceProperties struct {
Logging *Logging
HourMetrics *Metrics
MinuteMetrics *Metrics
Cors *Cors
}
input := StorageServiceProperties{
Logging: props.Logging,
HourMetrics: props.HourMetrics,
MinuteMetrics: props.MinuteMetrics,
Cors: props.Cors,
}
body, length, err := xmlMarshal(input)
if err != nil {
return err
}
headers := c.getStandardHeaders()
headers["Content-Length"] = strconv.Itoa(length)
resp, err := c.exec(http.MethodPut, uri, headers, body, auth)
if err != nil {
return err
}
readAndCloseBody(resp.body)
return checkRespCode(resp.statusCode, []int{http.StatusAccepted})
}

View file

@ -0,0 +1,412 @@
package storage
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
tablesURIPath = "/Tables"
nextTableQueryParameter = "NextTableName"
headerNextPartitionKey = "x-ms-continuation-NextPartitionKey"
headerNextRowKey = "x-ms-continuation-NextRowKey"
nextPartitionKeyQueryParameter = "NextPartitionKey"
nextRowKeyQueryParameter = "NextRowKey"
)
// TableAccessPolicy are used for SETTING table policies
type TableAccessPolicy struct {
ID string
StartTime time.Time
ExpiryTime time.Time
CanRead bool
CanAppend bool
CanUpdate bool
CanDelete bool
}
// Table represents an Azure table.
type Table struct {
tsc *TableServiceClient
Name string `json:"TableName"`
OdataEditLink string `json:"odata.editLink"`
OdataID string `json:"odata.id"`
OdataMetadata string `json:"odata.metadata"`
OdataType string `json:"odata.type"`
}
// EntityQueryResult contains the response from
// ExecuteQuery and ExecuteQueryNextResults functions.
type EntityQueryResult struct {
OdataMetadata string `json:"odata.metadata"`
Entities []*Entity `json:"value"`
QueryNextLink
table *Table
}
type continuationToken struct {
NextPartitionKey string
NextRowKey string
}
func (t *Table) buildPath() string {
return fmt.Sprintf("/%s", t.Name)
}
func (t *Table) buildSpecificPath() string {
return fmt.Sprintf("%s('%s')", tablesURIPath, t.Name)
}
// Get gets the referenced table.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/querying-tables-and-entities
func (t *Table) Get(timeout uint, ml MetadataLevel) error {
if ml == EmptyPayload {
return errEmptyPayload
}
query := url.Values{
"timeout": {strconv.FormatUint(uint64(timeout), 10)},
}
headers := t.tsc.client.getStandardHeaders()
headers[headerAccept] = string(ml)
uri := t.tsc.client.getEndpoint(tableServiceName, t.buildSpecificPath(), query)
resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return err
}
respBody, err := ioutil.ReadAll(resp.body)
if err != nil {
return err
}
err = json.Unmarshal(respBody, t)
if err != nil {
return err
}
return nil
}
// Create creates the referenced table.
// This function fails if the name is not compliant
// with the specification or the tables already exists.
// ml determines the level of detail of metadata in the operation response,
// or no data at all.
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/create-table
func (t *Table) Create(timeout uint, ml MetadataLevel, options *TableOptions) error {
uri := t.tsc.client.getEndpoint(tableServiceName, tablesURIPath, url.Values{
"timeout": {strconv.FormatUint(uint64(timeout), 10)},
})
type createTableRequest struct {
TableName string `json:"TableName"`
}
req := createTableRequest{TableName: t.Name}
buf := new(bytes.Buffer)
if err := json.NewEncoder(buf).Encode(req); err != nil {
return err
}
headers := t.tsc.client.getStandardHeaders()
headers = addReturnContentHeaders(headers, ml)
headers = addBodyRelatedHeaders(headers, buf.Len())
headers = options.addToHeaders(headers)
resp, err := t.tsc.client.exec(http.MethodPost, uri, headers, buf, t.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if ml == EmptyPayload {
if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
} else {
if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil {
return err
}
}
if ml != EmptyPayload {
data, err := ioutil.ReadAll(resp.body)
if err != nil {
return err
}
err = json.Unmarshal(data, t)
if err != nil {
return err
}
}
return nil
}
// Delete deletes the referenced table.
// This function fails if the table is not present.
// Be advised: Delete deletes all the entries that may be present.
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/delete-table
func (t *Table) Delete(timeout uint, options *TableOptions) error {
uri := t.tsc.client.getEndpoint(tableServiceName, t.buildSpecificPath(), url.Values{
"timeout": {strconv.Itoa(int(timeout))},
})
headers := t.tsc.client.getStandardHeaders()
headers = addReturnContentHeaders(headers, EmptyPayload)
headers = options.addToHeaders(headers)
resp, err := t.tsc.client.exec(http.MethodDelete, uri, headers, nil, t.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return nil
}
// QueryOptions includes options for a query entities operation.
// Top, filter and select are OData query options.
type QueryOptions struct {
Top uint
Filter string
Select []string
RequestID string
}
func (options *QueryOptions) getParameters() (url.Values, map[string]string) {
query := url.Values{}
headers := map[string]string{}
if options != nil {
if options.Top > 0 {
query.Add(OdataTop, strconv.FormatUint(uint64(options.Top), 10))
}
if options.Filter != "" {
query.Add(OdataFilter, options.Filter)
}
if len(options.Select) > 0 {
query.Add(OdataSelect, strings.Join(options.Select, ","))
}
headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
}
return query, headers
}
// QueryEntities returns the entities in the table.
// You can use query options defined by the OData Protocol specification.
//
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
func (t *Table) QueryEntities(timeout uint, ml MetadataLevel, options *QueryOptions) (*EntityQueryResult, error) {
if ml == EmptyPayload {
return nil, errEmptyPayload
}
query, headers := options.getParameters()
query = addTimeout(query, timeout)
uri := t.tsc.client.getEndpoint(tableServiceName, t.buildPath(), query)
return t.queryEntities(uri, headers, ml)
}
// NextResults returns the next page of results
// from a QueryEntities or NextResults operation.
//
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-timeout-and-pagination
func (eqr *EntityQueryResult) NextResults(options *TableOptions) (*EntityQueryResult, error) {
if eqr == nil {
return nil, errNilPreviousResult
}
if eqr.NextLink == nil {
return nil, errNilNextLink
}
headers := options.addToHeaders(map[string]string{})
return eqr.table.queryEntities(*eqr.NextLink, headers, eqr.ml)
}
// SetPermissions sets up table ACL permissions
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/Set-Table-ACL
func (t *Table) SetPermissions(tap []TableAccessPolicy, timeout uint, options *TableOptions) error {
params := url.Values{"comp": {"acl"},
"timeout": {strconv.Itoa(int(timeout))},
}
uri := t.tsc.client.getEndpoint(tableServiceName, t.Name, params)
headers := t.tsc.client.getStandardHeaders()
headers = options.addToHeaders(headers)
body, length, err := generateTableACLPayload(tap)
if err != nil {
return err
}
headers["Content-Length"] = strconv.Itoa(length)
resp, err := t.tsc.client.exec(http.MethodPut, uri, headers, body, t.tsc.auth)
if err != nil {
return err
}
defer readAndCloseBody(resp.body)
if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil {
return err
}
return nil
}
func generateTableACLPayload(policies []TableAccessPolicy) (io.Reader, int, error) {
sil := SignedIdentifiers{
SignedIdentifiers: []SignedIdentifier{},
}
for _, tap := range policies {
permission := generateTablePermissions(&tap)
signedIdentifier := convertAccessPolicyToXMLStructs(tap.ID, tap.StartTime, tap.ExpiryTime, permission)
sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
}
return xmlMarshal(sil)
}
// GetPermissions gets the table ACL permissions
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/get-table-acl
func (t *Table) GetPermissions(timeout int, options *TableOptions) ([]TableAccessPolicy, error) {
params := url.Values{"comp": {"acl"},
"timeout": {strconv.Itoa(int(timeout))},
}
uri := t.tsc.client.getEndpoint(tableServiceName, t.Name, params)
headers := t.tsc.client.getStandardHeaders()
headers = options.addToHeaders(headers)
resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
var ap AccessPolicy
err = xmlUnmarshal(resp.body, &ap.SignedIdentifiersList)
if err != nil {
return nil, err
}
return updateTableAccessPolicy(ap), nil
}
func (t *Table) queryEntities(uri string, headers map[string]string, ml MetadataLevel) (*EntityQueryResult, error) {
headers = mergeHeaders(headers, t.tsc.client.getStandardHeaders())
if ml != EmptyPayload {
headers[headerAccept] = string(ml)
}
resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
data, err := ioutil.ReadAll(resp.body)
if err != nil {
return nil, err
}
var entities EntityQueryResult
err = json.Unmarshal(data, &entities)
if err != nil {
return nil, err
}
for i := range entities.Entities {
entities.Entities[i].Table = t
}
entities.table = t
contToken := extractContinuationTokenFromHeaders(resp.headers)
if contToken == nil {
entities.NextLink = nil
} else {
originalURI, err := url.Parse(uri)
if err != nil {
return nil, err
}
v := originalURI.Query()
v.Set(nextPartitionKeyQueryParameter, contToken.NextPartitionKey)
v.Set(nextRowKeyQueryParameter, contToken.NextRowKey)
newURI := t.tsc.client.getEndpoint(tableServiceName, t.buildPath(), v)
entities.NextLink = &newURI
entities.ml = ml
}
return &entities, nil
}
func extractContinuationTokenFromHeaders(h http.Header) *continuationToken {
ct := continuationToken{
NextPartitionKey: h.Get(headerNextPartitionKey),
NextRowKey: h.Get(headerNextRowKey),
}
if ct.NextPartitionKey != "" && ct.NextRowKey != "" {
return &ct
}
return nil
}
func updateTableAccessPolicy(ap AccessPolicy) []TableAccessPolicy {
taps := []TableAccessPolicy{}
for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
tap := TableAccessPolicy{
ID: policy.ID,
StartTime: policy.AccessPolicy.StartTime,
ExpiryTime: policy.AccessPolicy.ExpiryTime,
}
tap.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
tap.CanAppend = updatePermissions(policy.AccessPolicy.Permission, "a")
tap.CanUpdate = updatePermissions(policy.AccessPolicy.Permission, "u")
tap.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d")
taps = append(taps, tap)
}
return taps
}
func generateTablePermissions(tap *TableAccessPolicy) (permissions string) {
// generate the permissions string (raud).
// still want the end user API to have bool flags.
permissions = ""
if tap.CanRead {
permissions += "r"
}
if tap.CanAppend {
permissions += "a"
}
if tap.CanUpdate {
permissions += "u"
}
if tap.CanDelete {
permissions += "d"
}
return permissions
}

View file

@ -0,0 +1,302 @@
package storage
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"sort"
"strings"
"github.com/satori/uuid"
)
// Operation type. Insert, Delete, Replace etc.
type Operation int
// consts for batch operations.
const (
InsertOp = Operation(1)
DeleteOp = Operation(2)
ReplaceOp = Operation(3)
MergeOp = Operation(4)
InsertOrReplaceOp = Operation(5)
InsertOrMergeOp = Operation(6)
)
// BatchEntity used for tracking Entities to operate on and
// whether operations (replace/merge etc) should be forced.
// Wrapper for regular Entity with additional data specific for the entity.
type BatchEntity struct {
*Entity
Force bool
Op Operation
}
// TableBatch stores all the entities that will be operated on during a batch process.
// Entities can be inserted, replaced or deleted.
type TableBatch struct {
BatchEntitySlice []BatchEntity
// reference to table we're operating on.
Table *Table
}
// defaultChangesetHeaders for changeSets
var defaultChangesetHeaders = map[string]string{
"Accept": "application/json;odata=minimalmetadata",
"Content-Type": "application/json",
"Prefer": "return-no-content",
}
// NewBatch return new TableBatch for populating.
func (t *Table) NewBatch() *TableBatch {
return &TableBatch{
Table: t,
}
}
// InsertEntity adds an entity in preparation for a batch insert.
func (t *TableBatch) InsertEntity(entity *Entity) {
be := BatchEntity{Entity: entity, Force: false, Op: InsertOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// InsertOrReplaceEntity adds an entity in preparation for a batch insert or replace.
func (t *TableBatch) InsertOrReplaceEntity(entity *Entity, force bool) {
be := BatchEntity{Entity: entity, Force: false, Op: InsertOrReplaceOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// InsertOrReplaceEntityByForce adds an entity in preparation for a batch insert or replace. Forces regardless of ETag
func (t *TableBatch) InsertOrReplaceEntityByForce(entity *Entity) {
t.InsertOrReplaceEntity(entity, true)
}
// InsertOrMergeEntity adds an entity in preparation for a batch insert or merge.
func (t *TableBatch) InsertOrMergeEntity(entity *Entity, force bool) {
be := BatchEntity{Entity: entity, Force: false, Op: InsertOrMergeOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// InsertOrMergeEntityByForce adds an entity in preparation for a batch insert or merge. Forces regardless of ETag
func (t *TableBatch) InsertOrMergeEntityByForce(entity *Entity) {
t.InsertOrMergeEntity(entity, true)
}
// ReplaceEntity adds an entity in preparation for a batch replace.
func (t *TableBatch) ReplaceEntity(entity *Entity) {
be := BatchEntity{Entity: entity, Force: false, Op: ReplaceOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// DeleteEntity adds an entity in preparation for a batch delete
func (t *TableBatch) DeleteEntity(entity *Entity, force bool) {
be := BatchEntity{Entity: entity, Force: false, Op: DeleteOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// DeleteEntityByForce adds an entity in preparation for a batch delete. Forces regardless of ETag
func (t *TableBatch) DeleteEntityByForce(entity *Entity, force bool) {
t.DeleteEntity(entity, true)
}
// MergeEntity adds an entity in preparation for a batch merge
func (t *TableBatch) MergeEntity(entity *Entity) {
be := BatchEntity{Entity: entity, Force: false, Op: MergeOp}
t.BatchEntitySlice = append(t.BatchEntitySlice, be)
}
// ExecuteBatch executes many table operations in one request to Azure.
// The operations can be combinations of Insert, Delete, Replace and Merge
// Creates the inner changeset body (various operations, Insert, Delete etc) then creates the outer request packet that encompasses
// the changesets.
// As per document https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/performing-entity-group-transactions
func (t *TableBatch) ExecuteBatch() error {
changesetBoundary := fmt.Sprintf("changeset_%s", uuid.NewV1())
uri := t.Table.tsc.client.getEndpoint(tableServiceName, "$batch", nil)
changesetBody, err := t.generateChangesetBody(changesetBoundary)
if err != nil {
return err
}
boundary := fmt.Sprintf("batch_%s", uuid.NewV1())
body, err := generateBody(changesetBody, changesetBoundary, boundary)
if err != nil {
return err
}
headers := t.Table.tsc.client.getStandardHeaders()
headers[headerContentType] = fmt.Sprintf("multipart/mixed; boundary=%s", boundary)
resp, err := t.Table.tsc.client.execBatchOperationJSON(http.MethodPost, uri, headers, bytes.NewReader(body.Bytes()), t.Table.tsc.auth)
if err != nil {
return err
}
defer resp.body.Close()
if err = checkRespCode(resp.statusCode, []int{http.StatusAccepted}); err != nil {
// check which batch failed.
operationFailedMessage := t.getFailedOperation(resp.odata.Err.Message.Value)
requestID, date, version := getDebugHeaders(resp.headers)
return AzureStorageServiceError{
StatusCode: resp.statusCode,
Code: resp.odata.Err.Code,
RequestID: requestID,
Date: date,
APIVersion: version,
Message: operationFailedMessage,
}
}
return nil
}
// getFailedOperation parses the original Azure error string and determines which operation failed
// and generates appropriate message.
func (t *TableBatch) getFailedOperation(errorMessage string) string {
// errorMessage consists of "number:string" we just need the number.
sp := strings.Split(errorMessage, ":")
if len(sp) > 1 {
msg := fmt.Sprintf("Element %s in the batch returned an unexpected response code.\n%s", sp[0], errorMessage)
return msg
}
// cant parse the message, just return the original message to client
return errorMessage
}
// generateBody generates the complete body for the batch request.
func generateBody(changeSetBody *bytes.Buffer, changesetBoundary string, boundary string) (*bytes.Buffer, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
writer.SetBoundary(boundary)
h := make(textproto.MIMEHeader)
h.Set(headerContentType, fmt.Sprintf("multipart/mixed; boundary=%s\r\n", changesetBoundary))
batchWriter, err := writer.CreatePart(h)
if err != nil {
return nil, err
}
batchWriter.Write(changeSetBody.Bytes())
writer.Close()
return body, nil
}
// generateChangesetBody generates the individual changesets for the various operations within the batch request.
// There is a changeset for Insert, Delete, Merge etc.
func (t *TableBatch) generateChangesetBody(changesetBoundary string) (*bytes.Buffer, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
writer.SetBoundary(changesetBoundary)
for _, be := range t.BatchEntitySlice {
t.generateEntitySubset(&be, writer)
}
writer.Close()
return body, nil
}
// generateVerb generates the HTTP request VERB required for each changeset.
func generateVerb(op Operation) (string, error) {
switch op {
case InsertOp:
return http.MethodPost, nil
case DeleteOp:
return http.MethodDelete, nil
case ReplaceOp, InsertOrReplaceOp:
return http.MethodPut, nil
case MergeOp, InsertOrMergeOp:
return "MERGE", nil
default:
return "", errors.New("Unable to detect operation")
}
}
// generateQueryPath generates the query path for within the changesets
// For inserts it will just be a table query path (table name)
// but for other operations (modifying an existing entity) then
// the partition/row keys need to be generated.
func (t *TableBatch) generateQueryPath(op Operation, entity *Entity) string {
if op == InsertOp {
return entity.Table.buildPath()
}
return entity.buildPath()
}
// generateGenericOperationHeaders generates common headers for a given operation.
func generateGenericOperationHeaders(be *BatchEntity) map[string]string {
retval := map[string]string{}
for k, v := range defaultChangesetHeaders {
retval[k] = v
}
if be.Op == DeleteOp || be.Op == ReplaceOp || be.Op == MergeOp {
if be.Force || be.Entity.OdataEtag == "" {
retval["If-Match"] = "*"
} else {
retval["If-Match"] = be.Entity.OdataEtag
}
}
return retval
}
// generateEntitySubset generates body payload for particular batch entity
func (t *TableBatch) generateEntitySubset(batchEntity *BatchEntity, writer *multipart.Writer) error {
h := make(textproto.MIMEHeader)
h.Set(headerContentType, "application/http")
h.Set(headerContentTransferEncoding, "binary")
verb, err := generateVerb(batchEntity.Op)
if err != nil {
return err
}
genericOpHeadersMap := generateGenericOperationHeaders(batchEntity)
queryPath := t.generateQueryPath(batchEntity.Op, batchEntity.Entity)
uri := t.Table.tsc.client.getEndpoint(tableServiceName, queryPath, nil)
operationWriter, err := writer.CreatePart(h)
if err != nil {
return err
}
urlAndVerb := fmt.Sprintf("%s %s HTTP/1.1\r\n", verb, uri)
operationWriter.Write([]byte(urlAndVerb))
writeHeaders(genericOpHeadersMap, &operationWriter)
operationWriter.Write([]byte("\r\n")) // additional \r\n is needed per changeset separating the "headers" and the body.
// delete operation doesn't need a body.
if batchEntity.Op != DeleteOp {
//var e Entity = batchEntity.Entity
body, err := json.Marshal(batchEntity.Entity)
if err != nil {
return err
}
operationWriter.Write(body)
}
return nil
}
func writeHeaders(h map[string]string, writer *io.Writer) {
// This way it is guaranteed the headers will be written in a sorted order
var keys []string
for k := range h {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
(*writer).Write([]byte(fmt.Sprintf("%s: %s\r\n", k, h[k])))
}
}

View file

@ -0,0 +1,190 @@
package storage
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
)
const (
headerAccept = "Accept"
headerEtag = "Etag"
headerPrefer = "Prefer"
headerXmsContinuation = "x-ms-Continuation-NextTableName"
)
// TableServiceClient contains operations for Microsoft Azure Table Storage
// Service.
type TableServiceClient struct {
client Client
auth authentication
}
// TableOptions includes options for some table operations
type TableOptions struct {
RequestID string
}
func (options *TableOptions) addToHeaders(h map[string]string) map[string]string {
if options != nil {
h = addToHeaders(h, "x-ms-client-request-id", options.RequestID)
}
return h
}
// QueryNextLink includes information for getting the next page of
// results in query operations
type QueryNextLink struct {
NextLink *string
ml MetadataLevel
}
// GetServiceProperties gets the properties of your storage account's table service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-table-service-properties
func (t *TableServiceClient) GetServiceProperties() (*ServiceProperties, error) {
return t.client.getServiceProperties(tableServiceName, t.auth)
}
// SetServiceProperties sets the properties of your storage account's table service.
// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-table-service-properties
func (t *TableServiceClient) SetServiceProperties(props ServiceProperties) error {
return t.client.setServiceProperties(props, tableServiceName, t.auth)
}
// GetTableReference returns a Table object for the specified table name.
func (t *TableServiceClient) GetTableReference(name string) *Table {
return &Table{
tsc: t,
Name: name,
}
}
// QueryTablesOptions includes options for some table operations
type QueryTablesOptions struct {
Top uint
Filter string
RequestID string
}
func (options *QueryTablesOptions) getParameters() (url.Values, map[string]string) {
query := url.Values{}
headers := map[string]string{}
if options != nil {
if options.Top > 0 {
query.Add(OdataTop, strconv.FormatUint(uint64(options.Top), 10))
}
if options.Filter != "" {
query.Add(OdataFilter, options.Filter)
}
headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
}
return query, headers
}
// QueryTables returns the tables in the storage account.
// You can use query options defined by the OData Protocol specification.
//
// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-tables
func (t *TableServiceClient) QueryTables(ml MetadataLevel, options *QueryTablesOptions) (*TableQueryResult, error) {
query, headers := options.getParameters()
uri := t.client.getEndpoint(tableServiceName, tablesURIPath, query)
return t.queryTables(uri, headers, ml)
}
// NextResults returns the next page of results
// from a QueryTables or a NextResults operation.
//
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-tables
// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-timeout-and-pagination
func (tqr *TableQueryResult) NextResults(options *TableOptions) (*TableQueryResult, error) {
if tqr == nil {
return nil, errNilPreviousResult
}
if tqr.NextLink == nil {
return nil, errNilNextLink
}
headers := options.addToHeaders(map[string]string{})
return tqr.tsc.queryTables(*tqr.NextLink, headers, tqr.ml)
}
// TableQueryResult contains the response from
// QueryTables and QueryTablesNextResults functions.
type TableQueryResult struct {
OdataMetadata string `json:"odata.metadata"`
Tables []Table `json:"value"`
QueryNextLink
tsc *TableServiceClient
}
func (t *TableServiceClient) queryTables(uri string, headers map[string]string, ml MetadataLevel) (*TableQueryResult, error) {
if ml == EmptyPayload {
return nil, errEmptyPayload
}
headers = mergeHeaders(headers, t.client.getStandardHeaders())
headers[headerAccept] = string(ml)
resp, err := t.client.exec(http.MethodGet, uri, headers, nil, t.auth)
if err != nil {
return nil, err
}
defer resp.body.Close()
if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil {
return nil, err
}
respBody, err := ioutil.ReadAll(resp.body)
if err != nil {
return nil, err
}
var out TableQueryResult
err = json.Unmarshal(respBody, &out)
if err != nil {
return nil, err
}
for i := range out.Tables {
out.Tables[i].tsc = t
}
out.tsc = t
nextLink := resp.headers.Get(http.CanonicalHeaderKey(headerXmsContinuation))
if nextLink == "" {
out.NextLink = nil
} else {
originalURI, err := url.Parse(uri)
if err != nil {
return nil, err
}
v := originalURI.Query()
v.Set(nextTableQueryParameter, nextLink)
newURI := t.client.getEndpoint(tableServiceName, tablesURIPath, v)
out.NextLink = &newURI
out.ml = ml
}
return &out, nil
}
func addBodyRelatedHeaders(h map[string]string, length int) map[string]string {
h[headerContentType] = "application/json"
h[headerContentLength] = fmt.Sprintf("%v", length)
h[headerAcceptCharset] = "UTF-8"
return h
}
func addReturnContentHeaders(h map[string]string, ml MetadataLevel) map[string]string {
if ml != EmptyPayload {
h[headerPrefer] = "return-content"
h[headerAccept] = string(ml)
} else {
h[headerPrefer] = "return-no-content"
// From API version 2015-12-11 onwards, Accept header is required
h[headerAccept] = string(NoMetadata)
}
return h
}

View file

@ -0,0 +1,199 @@
package storage
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"
"time"
)
var (
fixedTime = time.Date(2050, time.December, 20, 21, 55, 0, 0, time.FixedZone("GMT", -6))
)
func (c Client) computeHmac256(message string) string {
h := hmac.New(sha256.New, c.accountKey)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func currentTimeRfc1123Formatted() string {
return timeRfc1123Formatted(time.Now().UTC())
}
func timeRfc1123Formatted(t time.Time) string {
return t.Format(http.TimeFormat)
}
func mergeParams(v1, v2 url.Values) url.Values {
out := url.Values{}
for k, v := range v1 {
out[k] = v
}
for k, v := range v2 {
vals, ok := out[k]
if ok {
vals = append(vals, v...)
out[k] = vals
} else {
out[k] = v
}
}
return out
}
func prepareBlockListRequest(blocks []Block) string {
s := `<?xml version="1.0" encoding="utf-8"?><BlockList>`
for _, v := range blocks {
s += fmt.Sprintf("<%s>%s</%s>", v.Status, v.ID, v.Status)
}
s += `</BlockList>`
return s
}
func xmlUnmarshal(body io.Reader, v interface{}) error {
data, err := ioutil.ReadAll(body)
if err != nil {
return err
}
return xml.Unmarshal(data, v)
}
func xmlMarshal(v interface{}) (io.Reader, int, error) {
b, err := xml.Marshal(v)
if err != nil {
return nil, 0, err
}
return bytes.NewReader(b), len(b), nil
}
func headersFromStruct(v interface{}) map[string]string {
headers := make(map[string]string)
value := reflect.ValueOf(v)
for i := 0; i < value.NumField(); i++ {
key := value.Type().Field(i).Tag.Get("header")
if key != "" {
reflectedValue := reflect.Indirect(value.Field(i))
var val string
if reflectedValue.IsValid() {
switch reflectedValue.Type() {
case reflect.TypeOf(fixedTime):
val = timeRfc1123Formatted(reflectedValue.Interface().(time.Time))
case reflect.TypeOf(uint64(0)), reflect.TypeOf(uint(0)):
val = strconv.FormatUint(reflectedValue.Uint(), 10)
case reflect.TypeOf(int(0)):
val = strconv.FormatInt(reflectedValue.Int(), 10)
default:
val = reflectedValue.String()
}
}
if val != "" {
headers[key] = val
}
}
}
return headers
}
// merges extraHeaders into headers and returns headers
func mergeHeaders(headers, extraHeaders map[string]string) map[string]string {
for k, v := range extraHeaders {
headers[k] = v
}
return headers
}
func addToHeaders(h map[string]string, key, value string) map[string]string {
if value != "" {
h[key] = value
}
return h
}
func addTimeToHeaders(h map[string]string, key string, value *time.Time) map[string]string {
if value != nil {
h = addToHeaders(h, key, timeRfc1123Formatted(*value))
}
return h
}
func addTimeout(params url.Values, timeout uint) url.Values {
if timeout > 0 {
params.Add("timeout", fmt.Sprintf("%v", timeout))
}
return params
}
func addSnapshot(params url.Values, snapshot *time.Time) url.Values {
if snapshot != nil {
params.Add("snapshot", snapshot.Format("2006-01-02T15:04:05.0000000Z"))
}
return params
}
func getTimeFromHeaders(h http.Header, key string) (*time.Time, error) {
var out time.Time
var err error
outStr := h.Get(key)
if outStr != "" {
out, err = time.Parse(time.RFC1123, outStr)
if err != nil {
return nil, err
}
}
return &out, nil
}
// TimeRFC1123 is an alias for time.Time needed for custom Unmarshalling
type TimeRFC1123 time.Time
// UnmarshalXML is a custom unmarshaller that overrides the default time unmarshal which uses a different time layout.
func (t *TimeRFC1123) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var value string
d.DecodeElement(&value, &start)
parse, err := time.Parse(time.RFC1123, value)
if err != nil {
return err
}
*t = TimeRFC1123(parse)
return nil
}
// returns a map of custom metadata values from the specified HTTP header
func getMetadataFromHeaders(header http.Header) map[string]string {
metadata := make(map[string]string)
for k, v := range header {
// Can't trust CanonicalHeaderKey() to munge case
// reliably. "_" is allowed in identifiers:
// https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
// https://msdn.microsoft.com/library/aa664670(VS.71).aspx
// http://tools.ietf.org/html/rfc7230#section-3.2
// ...but "_" is considered invalid by
// CanonicalMIMEHeaderKey in
// https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542
// so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl".
k = strings.ToLower(k)
if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) {
continue
}
// metadata["lol"] = content of the last X-Ms-Meta-Lol header
k = k[len(userDefinedMetadataHeaderPrefix):]
metadata[k] = v[len(v)-1]
}
if len(metadata) == 0 {
return nil
}
return metadata
}

View file

@ -0,0 +1,12 @@
// +build !go1.8
package storage
import (
"io"
"net/http"
)
func setContentLengthFromLimitedReader(req *http.Request, lr *io.LimitedReader) {
req.ContentLength = lr.N
}

View file

@ -0,0 +1,18 @@
// +build go1.8
package storage
import (
"io"
"io/ioutil"
"net/http"
)
func setContentLengthFromLimitedReader(req *http.Request, lr *io.LimitedReader) {
req.ContentLength = lr.N
snapshot := *lr
req.GetBody = func() (io.ReadCloser, error) {
r := snapshot
return ioutil.NopCloser(&r), nil
}
}

View file

@ -0,0 +1,5 @@
package storage
var (
sdkVersion = "10.0.2"
)