diff --git a/cmd/umoci/gc.go b/cmd/umoci/gc.go index 45ad00905..19866372c 100644 --- a/cmd/umoci/gc.go +++ b/cmd/umoci/gc.go @@ -18,10 +18,13 @@ package main import ( + "regexp" + "github.com/openSUSE/umoci/oci/cas/dir" "github.com/openSUSE/umoci/oci/casext" "github.com/pkg/errors" "github.com/urfave/cli" + casDir "github.com/wking/casengine/dir" "golang.org/x/net/context" ) @@ -35,6 +38,12 @@ Where "" is the path to the OCI image. This command will do a mark-and-sweep garbage collection of the provided OCI image, only retaining blobs which can be reached by a descriptor path from the root set of references. All other blobs will be removed.`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "digest-regexp", + Usage: "regular expression for calculating the digest from a filesystem path. This is required if your oci-layout declares an oci-cas-template-v1 CAS engine (e.g. via 'umoci init --blob-uri ...')", + }, + }, // create modifies an image layout. Category: "layout", @@ -52,8 +61,22 @@ root set of references. All other blobs will be removed.`, func gc(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) + var getDigest casDir.GetDigest + if ctx.IsSet("digest-regexp") { + getDigestRegexp, err := regexp.Compile(ctx.String("digest-regexp")) + if err != nil { + return errors.Wrap(err, "compile digest-regexp") + } + + regexpGetDigest := &casDir.RegexpGetDigest{ + Regexp: getDigestRegexp, + } + + getDigest = regexpGetDigest.GetDigest + } + // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.OpenWithDigestLister(imagePath, getDigest) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/init.go b/cmd/umoci/init.go index f1c47e886..215e496a7 100644 --- a/cmd/umoci/init.go +++ b/cmd/umoci/init.go @@ -38,6 +38,13 @@ The new OCI image does not contain any references or blobs, but those can be created through the use of umoci-new(1), umoci-tag(1) and other similar commands.`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "blob-uri", + Usage: "URI Template for storing blobs, interpreted with the image path as the base URI. Defaults to blobs/{algorithm}/{encoded}", + }, + }, + // create modifies an image layout. Category: "layout", @@ -54,7 +61,7 @@ func initLayout(ctx *cli.Context) error { return errors.Wrap(err, "image layout creation") } - if err := dir.Create(imagePath); err != nil { + if err := dir.Create(imagePath, ctx.String("blob-uri")); err != nil { return errors.Wrap(err, "image layout creation") } diff --git a/doc/man/umoci-gc.1.md b/doc/man/umoci-gc.1.md index 432a4dd42..8bf067794 100644 --- a/doc/man/umoci-gc.1.md +++ b/doc/man/umoci-gc.1.md @@ -7,6 +7,7 @@ umoci gc - Garbage collects all unreferenced OCI image blobs # SYNOPSIS **umoci gc** **--layout**=*image* +[**--digest-regexp**=*regexp*] # DESCRIPTION Conduct a mark-and-sweep garbage collection of the provided OCI image, only @@ -20,6 +21,18 @@ The global options are defined in **umoci**(1). The OCI image layout to be garbage collected. *image* must be a path to a valid OCI image. +**--digest-regexp**=*regexp* + A regular expression for calculating the digest from a filesystem + path. This is required if your oci-layout declares an + `oci-cas-template-v1` CAS engine. For example, if you created the + image with: + + umoci init --blob-uri file:///path/to/my/blobs/{algorithm}/{encoded:2}/{encoded} + + Then you should set *regexp* to: + + ^.*/(?P[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P[a-zA-Z0-9=_-]{1,})$ + # EXAMPLE The following deletes a tag from an OCI image and clean conducts a garbage diff --git a/doc/man/umoci-init.1.md b/doc/man/umoci-init.1.md index 15235e0ad..ae0c7d5e7 100644 --- a/doc/man/umoci-init.1.md +++ b/doc/man/umoci-init.1.md @@ -7,6 +7,7 @@ umoci init - Create a new OCI image layout # SYNOPSIS **umoci init** **--layout**=*image* +[**--blob-uri**=*template* # DESCRIPTION Creates a new OCI image layout. The new OCI image does not contain any new @@ -21,6 +22,11 @@ The global options are defined in **umoci**(1). The path where the OCI image layout will be created. The path must not exist already or **umoci-init**(1) will return an error. +**--blob-uri**=*template* + The URI Template for retrieving digests. Relative URIs will be + resolved with the image path as the base URI. For more details, + see the [OCI CAS Template Protocol][cas-template]. + # EXAMPLE The following creates a brand new OCI image layout and then creates a blank tag @@ -33,3 +39,5 @@ for further manipulation with **umoci-repack**(1) and **umoci-config**(1). # SEE ALSO **umoci**(1), **umoci-new**(1) + +[cas-template]: https://github.com/xiekeyang/oci-discovery/blob/0be7eae246ae9a975a76ca209c045043f0793572/cas-template.md diff --git a/hack/vendor.sh b/hack/vendor.sh index 6d8dcaea8..033b8e4e9 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -131,8 +131,11 @@ clone github.com/pkg/errors v0.8.0 clone github.com/apex/log afb2e76037a5f36542c77e88ef8aef9f469b09f8 clone github.com/urfave/cli v1.20.0 clone github.com/cyphar/filepath-securejoin v0.2.1 -clone github.com/vbatts/go-mtree v0.4.1 -clone github.com/Sirupsen/logrus v1.0.3 +clone github.com/jtacoma/uritemplates v1.0.0 +clone github.com/vbatts/go-mtree 005af4d18f8ab74174ce23565be732a3101cf316 +clone github.com/sirupsen/logrus v1.0.3 +clone github.com/wking/casengine 3ed08888a9365a2753ab8b809b7efb286566fe8d +clone github.com/xiekeyang/oci-discovery 17aaa9ee7538d1db09b5f142ed319e06dee7407e clone golang.org/x/net 45e771701b814666a7eb299e6c7a57d0b1799e91 https://github.com/golang/net # Used purely for testing. clone github.com/mohae/deepcopy 491d3605edfb866af34a48075bd4355ac1bf46ca diff --git a/mutate/mutate_test.go b/mutate/mutate_test.go index 49f5b7424..b89d06bdc 100644 --- a/mutate/mutate_test.go +++ b/mutate/mutate_test.go @@ -47,7 +47,7 @@ const ( func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) { dir = filepath.Join(dir, "image") - if err := casdir.Create(dir); err != nil { + if err := casdir.Create(dir, ""); err != nil { t.Fatal(err) } diff --git a/oci/cas/cas.go b/oci/cas/cas.go index 423dfee40..c4611e88b 100644 --- a/oci/cas/cas.go +++ b/oci/cas/cas.go @@ -28,6 +28,7 @@ import ( "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/wking/casengine" "golang.org/x/net/context" ) @@ -63,13 +64,24 @@ var ( // Engine is an interface that provides methods for accessing and modifying an // OCI image, namely allowing access to reference descriptors and blobs. type Engine interface { + // CAS returns the casengine.Engine backing this engine. + CAS() (casEngine casengine.Engine) + + // DigestListerEngine returns the casengine.DigestListerEngine + // backing this engine, or nil if no such engine exists. + DigestListerEngine() (casEngine casengine.DigestListerEngine) + // PutBlob adds a new blob to the image. This is idempotent; a nil error // means that "the content is stored at DIGEST" without implying "because // of this PutBlob() call". + // + // Deprecated: Use CAS().Put instead. PutBlob(ctx context.Context, reader io.Reader) (digest digest.Digest, size int64, err error) // GetBlob returns a reader for retrieving a blob from the image, which the // caller must Close(). Returns ErrNotExist if the digest is not found. + // + // Deprecated: Use CAS().Get instead. GetBlob(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) // PutIndex sets the index of the OCI image to the given index, replacing @@ -92,9 +104,13 @@ type Engine interface { // DeleteBlob removes a blob from the image. This is idempotent; a nil // error means "the content is not in the store" without implying "because // of this DeleteBlob() call". + // + // Deprecated: Use CAS().Delete instead. DeleteBlob(ctx context.Context, digest digest.Digest) (err error) // ListBlobs returns the set of blob digests stored in the image. + // + // Deprecated: Use DigestListerEngine().Digests instead. ListBlobs(ctx context.Context) (digests []digest.Digest, err error) // Clean executes a garbage collection of any non-blob garbage in the store diff --git a/oci/cas/dir/cas_test.go b/oci/cas/dir/cas_test.go index 806158c4e..22ee5cb1d 100644 --- a/oci/cas/dir/cas_test.go +++ b/oci/cas/dir/cas_test.go @@ -19,14 +19,18 @@ package dir import ( "bytes" + "fmt" "io" "io/ioutil" "os" "path/filepath" + "regexp" "testing" "github.com/openSUSE/umoci/oci/cas" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/wking/casengine/dir" "golang.org/x/net/context" ) @@ -43,7 +47,8 @@ func TestCreateLayout(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -66,7 +71,7 @@ func TestCreateLayout(t *testing.T) { } // We should get an error if we try to create a new image atop an old one. - if err := Create(image); err == nil { + if err := Create(image, ""); err == nil { t.Errorf("expected to get a cowardly no-clobber error!") } } @@ -81,7 +86,7 @@ func TestEngineBlob(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -214,7 +219,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -234,7 +239,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -257,7 +262,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil { @@ -277,7 +282,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil { @@ -300,3 +305,73 @@ func TestEngineValidate(t *testing.T) { engine.Close() } } + +func TestEngineURITemplate(t *testing.T) { + ctx := context.Background() + + root, err := ioutil.TempDir("", "umoci-TestEngineURITemplate") + if err != nil { + t.Fatal(err) + } + //defer os.RemoveAll(root) + + image := filepath.Join(root, "image") + + if filepath.Separator != '/' { + t.Fatalf("CAS URI Template initialization is not implemented for filepath.Separator %q", filepath.Separator) + } + + if err := Create(image, fmt.Sprintf("file://%s/blobs/{algorithm}/{encoded:2}/{encoded}", root)); err != nil { + t.Fatalf("unexpected error creating image: %+v", err) + } + + getDigestRegexp, err := regexp.Compile(`^.*/blobs/(?P[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P[a-zA-Z0-9=_-]{1,})$`) + if err != nil { + t.Fatal(err) + } + + getDigest := &dir.RegexpGetDigest{ + Regexp: getDigestRegexp, + } + + engine, err := OpenWithDigestLister(image, getDigest.GetDigest) + if err != nil { + t.Fatalf("unexpected error opening image: %+v", err) + } + defer engine.Close() + + bytesIn := []byte("Hello, World!") + dig, err := engine.CAS().Put(ctx, digest.SHA256, bytes.NewReader(bytesIn)) + if err != nil { + t.Errorf("Put: unexpected error: %+v", err) + } + + reader, err := engine.CAS().Get(ctx, dig) + if err != nil { + t.Errorf("Get: unexpected error: %+v", err) + } + defer reader.Close() + + gotBytes, err := ioutil.ReadAll(reader) + if err != nil { + t.Errorf("Get: failed to ReadAll: %+v", err) + } + if !bytes.Equal(bytesIn, gotBytes) { + t.Errorf("Get: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes)) + } + + path := filepath.Join(root, "blobs", digest.SHA256.String(), "df", "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f") + reader, err = os.Open(path) + if err != nil { + t.Error(err) + } + defer reader.Close() + + gotBytes, err = ioutil.ReadAll(reader) + if err != nil { + t.Errorf("Open: failed to ReadAll: %+v", err) + } + if !bytes.Equal(bytesIn, gotBytes) { + t.Errorf("Open: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes)) + } +} diff --git a/oci/cas/dir/dir.go b/oci/cas/dir/dir.go index 8b83e4e57..a7d110378 100644 --- a/oci/cas/dir/dir.go +++ b/oci/cas/dir/dir.go @@ -19,10 +19,13 @@ package dir import ( "encoding/json" + "fmt" "io" "io/ioutil" "os" "path/filepath" + "regexp" + "strings" "github.com/apex/log" "github.com/openSUSE/umoci/oci/cas" @@ -30,18 +33,20 @@ import ( imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/wking/casengine" + "github.com/wking/casengine/counter" + "github.com/wking/casengine/dir" + "github.com/xiekeyang/oci-discovery/tools/refenginediscovery" "golang.org/x/net/context" "golang.org/x/sys/unix" ) const ( - // ImageLayoutVersion is the version of the image layout we support. This - // value is *not* the same as imagespec.Version, and the meaning of this - // field is still under discussion in the spec. For now we'll just hardcode - // the value and hope for the best. - ImageLayoutVersion = "1.0.0" - // blobDirectory is the directory inside an OCI image that contains blobs. + // + // FIXME: if the URI Template currently hard-coded Open() changes, + // then this variable will no longer be meaningful, and its consumers + // will have to be updated to use other logic. blobDirectory = "blobs" // indexFile is the file inside an OCI image that contains the top-level @@ -53,27 +58,12 @@ const ( layoutFile = "oci-layout" ) -// blobPath returns the path to a blob given its digest, relative to the root -// of the OCI image. The digest must be of the form algorithm:hex. -func blobPath(digest digest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", errors.Wrapf(err, "invalid digest: %q", digest) - } - - algo := digest.Algorithm() - hash := digest.Hex() - - if algo != cas.BlobAlgorithm { - return "", errors.Errorf("unsupported algorithm: %q", algo) - } - - return filepath.Join(blobDirectory, algo.String(), hash), nil -} - type dirEngine struct { - path string - temp string - tempFile *os.File + cas casengine.Engine + digestListerEngine casengine.DigestListerEngine + path string + temp string + tempFile *os.File } func (e *dirEngine) ensureTempDir() error { @@ -100,102 +90,35 @@ func (e *dirEngine) ensureTempDir() error { return nil } -// verify ensures that the image is valid. -func (e *dirEngine) validate() error { - content, err := ioutil.ReadFile(filepath.Join(e.path, layoutFile)) - if err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "read oci-layout") - } - - var ociLayout ispec.ImageLayout - if err := json.Unmarshal(content, &ociLayout); err != nil { - return errors.Wrap(err, "parse oci-layout") - } - - // XXX: Currently the meaning of this field is not adequately defined by - // the spec, nor is the "official" value determined by the spec. - if ociLayout.Version != ImageLayoutVersion { - return errors.Wrap(cas.ErrInvalid, "layout version is not supported") - } - - // Check that "blobs" and "index.json" exist in the image. - // FIXME: We also should check that blobs *only* contains a cas.BlobAlgorithm - // directory (with no subdirectories) and that refs *only* contains - // files (optionally also making sure they're all JSON descriptors). - if fi, err := os.Stat(filepath.Join(e.path, blobDirectory)); err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "check blobdir") - } else if !fi.IsDir() { - return errors.Wrap(cas.ErrInvalid, "blobdir is not a directory") - } - - if fi, err := os.Stat(filepath.Join(e.path, indexFile)); err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "check index") - } else if fi.IsDir() { - return errors.Wrap(cas.ErrInvalid, "index is a directory") - } +// CAS returns the casengine.Engine backing this engine. +func (e *dirEngine) CAS() (casEngine casengine.Engine) { + return e.cas +} - return nil +// DigestListerEngine returns the casengine.DigestListerEngine backing +// this engine. +func (e *dirEngine) DigestListerEngine() (casEngine casengine.DigestListerEngine) { + return e.digestListerEngine } // PutBlob adds a new blob to the image. This is idempotent; a nil error // means that "the content is stored at DIGEST" without implying "because // of this PutBlob() call". +// +// Deprecated: Use CAS().Put instead. func (e *dirEngine) PutBlob(ctx context.Context, reader io.Reader) (digest.Digest, int64, error) { - if err := e.ensureTempDir(); err != nil { - return "", -1, errors.Wrap(err, "ensure tempdir") - } - - digester := cas.BlobAlgorithm.Digester() - - // We copy this into a temporary file because we need to get the blob hash, - // but also to avoid half-writing an invalid blob. - fh, err := ioutil.TempFile(e.temp, "blob-") - if err != nil { - return "", -1, errors.Wrap(err, "create temporary blob") - } - tempPath := fh.Name() - defer fh.Close() - - writer := io.MultiWriter(fh, digester.Hash()) - size, err := io.Copy(writer, reader) - if err != nil { - return "", -1, errors.Wrap(err, "copy to temporary blob") - } - fh.Close() - - // Get the digest. - path, err := blobPath(digester.Digest()) - if err != nil { - return "", -1, errors.Wrap(err, "compute blob name") - } - - // Move the blob to its correct path. - path = filepath.Join(e.path, path) - if err := os.Rename(tempPath, path); err != nil { - return "", -1, errors.Wrap(err, "rename temporary blob") - } - - return digester.Digest(), int64(size), nil + counter := &counter.Counter{} + countedReader := io.TeeReader(reader, counter) + dig, err := e.cas.Put(ctx, cas.BlobAlgorithm, countedReader) + return dig, int64(counter.Count()), err } // GetBlob returns a reader for retrieving a blob from the image, which the // caller must Close(). Returns os.ErrNotExist if the digest is not found. +// +// Deprecated: Use CAS().Get instead. func (e *dirEngine) GetBlob(ctx context.Context, digest digest.Digest) (io.ReadCloser, error) { - path, err := blobPath(digest) - if err != nil { - return nil, errors.Wrap(err, "compute blob path") - } - fh, err := os.Open(filepath.Join(e.path, path)) - return fh, errors.Wrap(err, "open blob") + return e.cas.Get(ctx, digest) } // PutIndex sets the index of the OCI image to the given index, replacing the @@ -259,36 +182,27 @@ func (e *dirEngine) GetIndex(ctx context.Context) (ispec.Index, error) { // DeleteBlob removes a blob from the image. This is idempotent; a nil // error means "the content is not in the store" without implying "because // of this DeleteBlob() call". +// +// Deprecated: Use CAS().Delete instead. func (e *dirEngine) DeleteBlob(ctx context.Context, digest digest.Digest) error { - path, err := blobPath(digest) - if err != nil { - return errors.Wrap(err, "compute blob path") - } - - err = os.Remove(filepath.Join(e.path, path)) - if err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "remove blob") - } - return nil + return e.cas.Delete(ctx, digest) } // ListBlobs returns the set of blob digests stored in the image. +// +// Deprecated: Use DigestListerEngine().Digests instead. func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { - digests := []digest.Digest{} - blobDir := filepath.Join(e.path, blobDirectory, cas.BlobAlgorithm.String()) - - if err := filepath.Walk(blobDir, func(path string, _ os.FileInfo, _ error) error { - // Skip the actual directory. - if path == blobDir { - return nil - } + if e.digestListerEngine == nil { + return nil, fmt.Errorf("cannot list blobs without a DigestListerEngine") + } - // XXX: Do we need to handle multiple-directory-deep cases? - digest := digest.NewDigestFromHex(cas.BlobAlgorithm.String(), filepath.Base(path)) + digests := []digest.Digest{} + err := e.digestListerEngine.Digests(ctx, "", "", -1, 0, func(ctx context.Context, digest digest.Digest) (err error) { digests = append(digests, digest) return nil - }); err != nil { - return nil, errors.Wrap(err, "walk blobdir") + }) + if err != nil { + return nil, err } return digests, nil @@ -340,40 +254,174 @@ func (e *dirEngine) cleanPath(ctx context.Context, path string) error { // Close releases all references held by the e. Subsequent operations may // fail. -func (e *dirEngine) Close() error { +func (e *dirEngine) Close() (err error) { + ctx := context.Background() + var err2 error + if e.cas != nil { + if err2 = e.cas.Close(ctx); err2 != nil { + err = errors.Wrap(err, "close CAS") + } + } + if e.temp != "" { - if err := unix.Flock(int(e.tempFile.Fd()), unix.LOCK_UN); err != nil { - return errors.Wrap(err, "unlock tempdir") + if err2 := unix.Flock(int(e.tempFile.Fd()), unix.LOCK_UN); err2 != nil { + err2 = errors.Wrap(err2, "unlock tempdir") + if err == nil { + err = err2 + } } - if err := e.tempFile.Close(); err != nil { - return errors.Wrap(err, "close tempdir") + if err2 := e.tempFile.Close(); err2 != nil { + err2 = errors.Wrap(err2, "close tempdir") + if err == nil { + err = err2 + } } - if err := os.RemoveAll(e.temp); err != nil { - return errors.Wrap(err, "remove tempdir") + if err2 := os.RemoveAll(e.temp); err != nil { + err2 = errors.Wrap(err2, "remove tempdir") + if err == nil { + err = err2 + } } } - return nil + return err +} + +// Open opens a new reference to the directory-backed OCI image +// referenced by the provided path. If your image configures a custom +// blob URI, use OpenWithDigestLister instead. +func Open(path string) (engine cas.Engine, err error) { + return OpenWithDigestLister(path, nil) +} + +// OpenWithDigestLister opens a new reference to the directory-backed +// OCI image referenced by the provided path. Use this function +// instead of Open when your image configures a custom blob URI. +func OpenWithDigestLister(path string, getDigest dir.GetDigest) (engine cas.Engine, err error) { + ctx := context.Background() + + configBytes, err := ioutil.ReadFile(filepath.Join(path, layoutFile)) + if err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "read oci-layout") + } + + var ociLayout ispec.ImageLayout + if err := json.Unmarshal(configBytes, &ociLayout); err != nil { + return nil, errors.Wrap(err, "parse oci-layout") + } + + uri := "blobs/{algorithm}/{encoded}" + + // XXX: Currently the meaning of this field is not adequately defined by + // the spec, nor is the "official" value determined by the spec. + switch ociLayout.Version { + case "1.0.0": // nothing to configure here + case "1.1.0": + var engines refenginediscovery.Engines + if err := json.Unmarshal(configBytes, &engines); err != nil { + return nil, errors.Wrap(err, "parse oci-layout") + } + for _, config := range engines.CASEngines { + if config.Protocol == "oci-cas-template-v1" { + uriInterface, ok := config.Data["uri"] + if !ok { + return nil, fmt.Errorf("CAS-template config missing required 'uri' property: %v", config.Data) + } + + uri, ok = uriInterface.(string) + if !ok { + return nil, fmt.Errorf("CAS-template config 'uri' is not a string: %v", uriInterface) + } + + break + } + } + default: + return nil, errors.Wrap(cas.ErrInvalid, fmt.Sprintf("layout version %s is not supported", ociLayout.Version)) + } + + if uri == "blobs/{algorithm}/{encoded}" { + getDigest, err = defaultGetDigest() + if err != nil { + return nil, err + } + } + + var casEngine casengine.Engine + var digestListerEngine casengine.DigestListerEngine + if getDigest == nil { + casEngine, err = dir.NewEngine(ctx, path, uri) + if err != nil { + return nil, errors.Wrap(err, "initialize CAS engine") + } + } else { + digestListerEngine, err = dir.NewDigestListerEngine(ctx, path, uri, getDigest) + if err != nil { + return nil, errors.Wrap(err, "initialize CAS engine") + } + casEngine = digestListerEngine + } + defer func() { + if err != nil { + casEngine.Close(ctx) + } + }() + + // Check that "blobs" and "index.json" exist in the image. + if fi, err := os.Stat(filepath.Join(path, blobDirectory)); err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "check blobdir") + } else if !fi.IsDir() { + return nil, errors.Wrap(cas.ErrInvalid, "blobdir is not a directory") + } + + if fi, err := os.Stat(filepath.Join(path, indexFile)); err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "check index") + } else if fi.IsDir() { + return nil, errors.Wrap(cas.ErrInvalid, "index is a directory") + } + + return &dirEngine{ + cas: casEngine, + digestListerEngine: digestListerEngine, + path: path, + temp: "", + }, nil } -// Open opens a new reference to the directory-backed OCI image referenced by -// the provided path. -func Open(path string) (cas.Engine, error) { - engine := &dirEngine{ - path: path, - temp: "", +func defaultGetDigest() (getDigest dir.GetDigest, err error) { + pattern := `^blobs/(?P[a-z0-9+._-]+)/(?P[a-zA-Z0-9=_-]{1,})$` + if filepath.Separator != '/' { + if filepath.Separator == '\\' { + pattern = strings.Replace(pattern, "/", `\\`, -1) + } else { + return nil, fmt.Errorf("unknown path separator %q", string(filepath.Separator)) + } + } + + getDigestRegexp, err := regexp.Compile(pattern) + if err != nil { + return nil, errors.Wrap(err, "get-digest regexp") } - if err := engine.validate(); err != nil { - return nil, errors.Wrap(err, "validate") + regexpGetDigest := &dir.RegexpGetDigest{ + Regexp: getDigestRegexp, } - return engine, nil + return regexpGetDigest.GetDigest, nil } // Create creates a new OCI image layout at the given path. If the path already // exists, os.ErrExist is returned. However, all of the parent components of // the path will be created if necessary. -func Create(path string) error { +func Create(path string, uri string) error { // We need to fail if path already exists, but we first create all of the // parent paths. dir := filepath.Dir(path) @@ -415,8 +463,22 @@ func Create(path string) error { } defer layoutFh.Close() - ociLayout := ispec.ImageLayout{ - Version: ImageLayoutVersion, + var ociLayout interface{} + switch uri { + case "": + ociLayout = ispec.ImageLayout{ + Version: "1.0.0", + } + default: + ociLayout = map[string]interface{}{ + "imageLayoutVersion": "1.1.0", + "casEngines": []map[string]interface{}{ + { + "protocol": "oci-cas-template-v1", + "uri": uri, + }, + }, + } } if err := json.NewEncoder(layoutFh).Encode(ociLayout); err != nil { return errors.Wrap(err, "encode oci-layout") diff --git a/oci/cas/dir/dir_test.go b/oci/cas/dir/dir_test.go index 29b143137..9021943c9 100644 --- a/oci/cas/dir/dir_test.go +++ b/oci/cas/dir/dir_test.go @@ -23,9 +23,12 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "testing" "github.com/openSUSE/umoci/oci/cas" + imeta "github.com/opencontainers/image-spec/specs-go" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sys/unix" @@ -77,7 +80,7 @@ func TestCreateLayoutReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -114,7 +117,7 @@ func TestEngineBlobReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -202,7 +205,7 @@ func TestEngineGCLocking(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -232,8 +235,14 @@ func TestEngineGCLocking(t *testing.T) { t.Errorf("PutBlob: length doesn't match: expected=%d got=%d", len(content), size) } + err = engine.PutIndex(ctx, ispec.Index{ + Versioned: imeta.Versioned{ + SchemaVersion: 2, // FIXME: This is hardcoded at the moment. + }, + }) + if engine.(*dirEngine).temp == "" { - t.Errorf("engine doesn't have a tempdir after putting a blob!") + t.Errorf("engine doesn't have a tempdir after putting an index!") } // Create umoci and other directories and files to make sure things work. @@ -277,3 +286,39 @@ func TestEngineGCLocking(t *testing.T) { } } } + +// Check for error when getDigest is needed but not provided. +func TestEngineMissingGetDigest(t *testing.T) { + ctx := context.Background() + + root, err := ioutil.TempDir("", "umoci-TestCreateLayoutReadonly") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + + image := filepath.Join(root, "image") + if err := Create(image, "file:///some/where/{blob}/{algorithm:2}/{algorithm}"); err != nil { + t.Fatalf("unexpected error creating image: %+v", err) + } + + engine, err := OpenWithDigestLister(image, nil) + if err != nil { + t.Fatal(err) + } + + _, err = engine.ListBlobs(ctx) + if err == nil { + t.Fatal("open did not raise an error") + } + matched, err2 := regexp.MatchString( + `^cannot list blobs without a DigestListerEngine$`, + err.Error(), + ) + if err2 != nil { + t.Fatal(err2) + } + if !matched { + t.Fatalf("open did not raise the expected error: %s", err) + } +} diff --git a/oci/casext/json_dir_test.go b/oci/casext/json_dir_test.go index 3ec2072fd..a1fc16de1 100644 --- a/oci/casext/json_dir_test.go +++ b/oci/casext/json_dir_test.go @@ -40,7 +40,7 @@ func TestEngineBlobJSON(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -124,7 +124,7 @@ func TestEngineBlobJSONReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } diff --git a/oci/casext/refname_dir_test.go b/oci/casext/refname_dir_test.go index d1815e866..4e79cff0c 100644 --- a/oci/casext/refname_dir_test.go +++ b/oci/casext/refname_dir_test.go @@ -246,7 +246,7 @@ func TestEngineReference(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -310,7 +310,7 @@ func TestEngineReferenceReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } diff --git a/oci/layer/unpack_test.go b/oci/layer/unpack_test.go index 51148ca98..13170b3e4 100644 --- a/oci/layer/unpack_test.go +++ b/oci/layer/unpack_test.go @@ -88,7 +88,7 @@ yRAbACGEEEIIIYQQQgghhBBCCKEr+wTE0sQyACgAAA==`, // Create our image. image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatal(err) } engine, err := dir.Open(image) diff --git a/test/create.bats b/test/create.bats old mode 100644 new mode 100755 index 55b32a1a7..85c5cc565 --- a/test/create.bats +++ b/test/create.bats @@ -65,6 +65,40 @@ function teardown() { image-verify "$NEWIMAGE" } +@test "umoci init --blob-uri file://... --layout ..." { + # Setup up $NEWIMAGE. + NEWIMAGE="$(setup_tmpdir)" + rm -rf "$NEWIMAGE" + + # Create a separate directory for CAS blobs + CAS="$(setup_tmpdir)" + + # Create a new image with no tags. + umoci init --blob-uri "file://${CAS}" --layout "$NEWIMAGE" + [ "$status" -eq 0 ] + image-verify "$NEWIMAGE" + + # Make sure that there are no references or blobs. + sane_run find "$NEWIMAGE/blobs" -type f + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 0 ] + # Note that this is _very_ dodgy at the moment because of how complicated + # the reference handling is now. + # XXX: Make sure to update this for 1.0.0-rc6 where the refname changed. + sane_run jq -SMr '.manifests[]? | .annotations["org.opencontainers.ref.name"] | strings' "$NEWIMAGE/index.json" + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 0 ] + + # Make sure that the required files exist. + [ -f "$NEWIMAGE/oci-layout" ] + [ -d "$NEWIMAGE/blobs" ] + [ -f "$NEWIMAGE/index.json" ] + + # FIXME: check that oci-layout contains the expected casEngines. + + image-verify "$NEWIMAGE" +} + @test "umoci new [missing args]" { umoci new [ "$status" -ne 0 ] diff --git a/vendor/github.com/jtacoma/uritemplates/LICENSE b/vendor/github.com/jtacoma/uritemplates/LICENSE new file mode 100644 index 000000000..7109c6ef9 --- /dev/null +++ b/vendor/github.com/jtacoma/uritemplates/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Joshua Tacoma. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/jtacoma/uritemplates/uritemplates.go b/vendor/github.com/jtacoma/uritemplates/uritemplates.go new file mode 100644 index 000000000..fc746ebb8 --- /dev/null +++ b/vendor/github.com/jtacoma/uritemplates/uritemplates.go @@ -0,0 +1,380 @@ +// Copyright 2013 Joshua Tacoma. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package uritemplates is a level 4 implementation of RFC 6570 (URI +// Template, http://tools.ietf.org/html/rfc6570). +// +// To use uritemplates, parse a template string and expand it with a value +// map: +// +// template, _ := uritemplates.Parse("https://api.github.com/repos{/user,repo}") +// values := make(map[string]interface{}) +// values["user"] = "jtacoma" +// values["repo"] = "uritemplates" +// expanded, _ := template.Expand(values) +// fmt.Printf(expanded) +// +package uritemplates + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" +) + +var ( + unreserved = regexp.MustCompile("[^A-Za-z0-9\\-._~]") + reserved = regexp.MustCompile("[^A-Za-z0-9\\-._~:/?#[\\]@!$&'()*+,;=]") + validname = regexp.MustCompile("^([A-Za-z0-9_\\.]|%[0-9A-Fa-f][0-9A-Fa-f])+$") + hex = []byte("0123456789ABCDEF") +) + +func pctEncode(src []byte) []byte { + dst := make([]byte, len(src)*3) + for i, b := range src { + buf := dst[i*3 : i*3+3] + buf[0] = 0x25 + buf[1] = hex[b/16] + buf[2] = hex[b%16] + } + return dst +} + +func escape(s string, allowReserved bool) (escaped string) { + if allowReserved { + escaped = string(reserved.ReplaceAllFunc([]byte(s), pctEncode)) + } else { + escaped = string(unreserved.ReplaceAllFunc([]byte(s), pctEncode)) + } + return escaped +} + +// A UriTemplate is a parsed representation of a URI template. +type UriTemplate struct { + raw string + parts []templatePart +} + +// Parse parses a URI template string into a UriTemplate object. +func Parse(rawtemplate string) (template *UriTemplate, err error) { + template = new(UriTemplate) + template.raw = rawtemplate + split := strings.Split(rawtemplate, "{") + template.parts = make([]templatePart, len(split)*2-1) + for i, s := range split { + if i == 0 { + if strings.Contains(s, "}") { + err = errors.New("unexpected }") + break + } + template.parts[i].raw = s + } else { + subsplit := strings.Split(s, "}") + if len(subsplit) != 2 { + err = errors.New("malformed template") + break + } + expression := subsplit[0] + template.parts[i*2-1], err = parseExpression(expression) + if err != nil { + break + } + template.parts[i*2].raw = subsplit[1] + } + } + if err != nil { + template = nil + } + return template, err +} + +func (t UriTemplate) String() string { + return t.raw +} + +type templatePart struct { + raw string + terms []templateTerm + first string + sep string + named bool + ifemp string + allowReserved bool +} + +type templateTerm struct { + name string + explode bool + truncate int +} + +func parseExpression(expression string) (result templatePart, err error) { + switch expression[0] { + case '+': + result.sep = "," + result.allowReserved = true + expression = expression[1:] + case '.': + result.first = "." + result.sep = "." + expression = expression[1:] + case '/': + result.first = "/" + result.sep = "/" + expression = expression[1:] + case ';': + result.first = ";" + result.sep = ";" + result.named = true + expression = expression[1:] + case '?': + result.first = "?" + result.sep = "&" + result.named = true + result.ifemp = "=" + expression = expression[1:] + case '&': + result.first = "&" + result.sep = "&" + result.named = true + result.ifemp = "=" + expression = expression[1:] + case '#': + result.first = "#" + result.sep = "," + result.allowReserved = true + expression = expression[1:] + default: + result.sep = "," + } + rawterms := strings.Split(expression, ",") + result.terms = make([]templateTerm, len(rawterms)) + for i, raw := range rawterms { + result.terms[i], err = parseTerm(raw) + if err != nil { + break + } + } + return result, err +} + +func parseTerm(term string) (result templateTerm, err error) { + if strings.HasSuffix(term, "*") { + result.explode = true + term = term[:len(term)-1] + } + split := strings.Split(term, ":") + if len(split) == 1 { + result.name = term + } else if len(split) == 2 { + result.name = split[0] + var parsed int64 + parsed, err = strconv.ParseInt(split[1], 10, 0) + result.truncate = int(parsed) + } else { + err = errors.New("multiple colons in same term") + } + if !validname.MatchString(result.name) { + err = errors.New("not a valid name: " + result.name) + } + if result.explode && result.truncate > 0 { + err = errors.New("both explode and prefix modifers on same term") + } + return result, err +} + +// Names returns the names of all variables within the template. +func (self *UriTemplate) Names() []string { + names := make([]string, 0, len(self.parts)) + + for _, p := range self.parts { + if len(p.raw) > 0 || len(p.terms) == 0 { + continue + } + + for _, term := range p.terms { + names = append(names, term.name) + } + } + + return names +} + +// Expand expands a URI template with a set of values to produce a string. +func (self *UriTemplate) Expand(value interface{}) (string, error) { + values, ismap := value.(map[string]interface{}) + if !ismap { + if m, ismap := struct2map(value); !ismap { + return "", errors.New("expected map[string]interface{}, struct, or pointer to struct.") + } else { + return self.Expand(m) + } + } + var buf bytes.Buffer + for _, p := range self.parts { + err := p.expand(&buf, values) + if err != nil { + return "", err + } + } + return buf.String(), nil +} + +func (self *templatePart) expand(buf *bytes.Buffer, values map[string]interface{}) error { + if len(self.raw) > 0 { + buf.WriteString(self.raw) + return nil + } + var zeroLen = buf.Len() + buf.WriteString(self.first) + var firstLen = buf.Len() + for _, term := range self.terms { + value, exists := values[term.name] + if !exists { + continue + } + if buf.Len() != firstLen { + buf.WriteString(self.sep) + } + switch v := value.(type) { + case string: + self.expandString(buf, term, v) + case []interface{}: + self.expandArray(buf, term, v) + case map[string]interface{}: + if term.truncate > 0 { + return errors.New("cannot truncate a map expansion") + } + self.expandMap(buf, term, v) + default: + if m, ismap := struct2map(value); ismap { + if term.truncate > 0 { + return errors.New("cannot truncate a map expansion") + } + self.expandMap(buf, term, m) + } else { + str := fmt.Sprintf("%v", value) + self.expandString(buf, term, str) + } + } + } + if buf.Len() == firstLen { + original := buf.Bytes()[:zeroLen] + buf.Reset() + buf.Write(original) + } + return nil +} + +func (self *templatePart) expandName(buf *bytes.Buffer, name string, empty bool) { + if self.named { + buf.WriteString(name) + if empty { + buf.WriteString(self.ifemp) + } else { + buf.WriteString("=") + } + } +} + +func (self *templatePart) expandString(buf *bytes.Buffer, t templateTerm, s string) { + if len(s) > t.truncate && t.truncate > 0 { + s = s[:t.truncate] + } + self.expandName(buf, t.name, len(s) == 0) + buf.WriteString(escape(s, self.allowReserved)) +} + +func (self *templatePart) expandArray(buf *bytes.Buffer, t templateTerm, a []interface{}) { + if len(a) == 0 { + return + } else if !t.explode { + self.expandName(buf, t.name, false) + } + for i, value := range a { + if t.explode && i > 0 { + buf.WriteString(self.sep) + } else if i > 0 { + buf.WriteString(",") + } + var s string + switch v := value.(type) { + case string: + s = v + default: + s = fmt.Sprintf("%v", v) + } + if len(s) > t.truncate && t.truncate > 0 { + s = s[:t.truncate] + } + if self.named && t.explode { + self.expandName(buf, t.name, len(s) == 0) + } + buf.WriteString(escape(s, self.allowReserved)) + } +} + +func (self *templatePart) expandMap(buf *bytes.Buffer, t templateTerm, m map[string]interface{}) { + if len(m) == 0 { + return + } + if !t.explode { + self.expandName(buf, t.name, len(m) == 0) + } + var firstLen = buf.Len() + for k, value := range m { + if firstLen != buf.Len() { + if t.explode { + buf.WriteString(self.sep) + } else { + buf.WriteString(",") + } + } + var s string + switch v := value.(type) { + case string: + s = v + default: + s = fmt.Sprintf("%v", v) + } + if t.explode { + buf.WriteString(escape(k, self.allowReserved)) + buf.WriteRune('=') + buf.WriteString(escape(s, self.allowReserved)) + } else { + buf.WriteString(escape(k, self.allowReserved)) + buf.WriteRune(',') + buf.WriteString(escape(s, self.allowReserved)) + } + } +} + +func struct2map(v interface{}) (map[string]interface{}, bool) { + value := reflect.ValueOf(v) + switch value.Type().Kind() { + case reflect.Ptr: + return struct2map(value.Elem().Interface()) + case reflect.Struct: + m := make(map[string]interface{}) + for i := 0; i < value.NumField(); i++ { + tag := value.Type().Field(i).Tag + var name string + if strings.Contains(string(tag), ":") { + name = tag.Get("uri") + } else { + name = strings.TrimSpace(string(tag)) + } + if len(name) == 0 { + name = value.Type().Field(i).Name + } + m[name] = value.Field(i).Interface() + } + return m, true + } + return nil, false +} diff --git a/vendor/github.com/Sirupsen/logrus/LICENSE b/vendor/github.com/sirupsen/logrus/LICENSE similarity index 100% rename from vendor/github.com/Sirupsen/logrus/LICENSE rename to vendor/github.com/sirupsen/logrus/LICENSE diff --git a/vendor/github.com/Sirupsen/logrus/alt_exit.go b/vendor/github.com/sirupsen/logrus/alt_exit.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/alt_exit.go rename to vendor/github.com/sirupsen/logrus/alt_exit.go diff --git a/vendor/github.com/Sirupsen/logrus/doc.go b/vendor/github.com/sirupsen/logrus/doc.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/doc.go rename to vendor/github.com/sirupsen/logrus/doc.go diff --git a/vendor/github.com/Sirupsen/logrus/entry.go b/vendor/github.com/sirupsen/logrus/entry.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/entry.go rename to vendor/github.com/sirupsen/logrus/entry.go diff --git a/vendor/github.com/Sirupsen/logrus/exported.go b/vendor/github.com/sirupsen/logrus/exported.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/exported.go rename to vendor/github.com/sirupsen/logrus/exported.go diff --git a/vendor/github.com/Sirupsen/logrus/formatter.go b/vendor/github.com/sirupsen/logrus/formatter.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/formatter.go rename to vendor/github.com/sirupsen/logrus/formatter.go diff --git a/vendor/github.com/Sirupsen/logrus/hooks.go b/vendor/github.com/sirupsen/logrus/hooks.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/hooks.go rename to vendor/github.com/sirupsen/logrus/hooks.go diff --git a/vendor/github.com/Sirupsen/logrus/json_formatter.go b/vendor/github.com/sirupsen/logrus/json_formatter.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/json_formatter.go rename to vendor/github.com/sirupsen/logrus/json_formatter.go diff --git a/vendor/github.com/Sirupsen/logrus/logger.go b/vendor/github.com/sirupsen/logrus/logger.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/logger.go rename to vendor/github.com/sirupsen/logrus/logger.go diff --git a/vendor/github.com/Sirupsen/logrus/logrus.go b/vendor/github.com/sirupsen/logrus/logrus.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/logrus.go rename to vendor/github.com/sirupsen/logrus/logrus.go diff --git a/vendor/github.com/Sirupsen/logrus/terminal_bsd.go b/vendor/github.com/sirupsen/logrus/terminal_bsd.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/terminal_bsd.go rename to vendor/github.com/sirupsen/logrus/terminal_bsd.go diff --git a/vendor/github.com/Sirupsen/logrus/terminal_linux.go b/vendor/github.com/sirupsen/logrus/terminal_linux.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/terminal_linux.go rename to vendor/github.com/sirupsen/logrus/terminal_linux.go diff --git a/vendor/github.com/Sirupsen/logrus/text_formatter.go b/vendor/github.com/sirupsen/logrus/text_formatter.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/text_formatter.go rename to vendor/github.com/sirupsen/logrus/text_formatter.go diff --git a/vendor/github.com/Sirupsen/logrus/writer.go b/vendor/github.com/sirupsen/logrus/writer.go similarity index 100% rename from vendor/github.com/Sirupsen/logrus/writer.go rename to vendor/github.com/sirupsen/logrus/writer.go diff --git a/vendor/github.com/vbatts/go-mtree/keywordfunc.go b/vendor/github.com/vbatts/go-mtree/keywordfunc.go index a3ef6e469..7a8a17003 100644 --- a/vendor/github.com/vbatts/go-mtree/keywordfunc.go +++ b/vendor/github.com/vbatts/go-mtree/keywordfunc.go @@ -26,30 +26,32 @@ type KeywordFunc func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, err var ( // KeywordFuncs is the map of all keywords (and the functions to produce them) KeywordFuncs = map[Keyword]KeywordFunc{ - "size": sizeKeywordFunc, // The size, in bytes, of the file - "type": typeKeywordFunc, // The type of the file - "time": timeKeywordFunc, // The last modification time of the file - "link": linkKeywordFunc, // The target of the symbolic link when type=link - "uid": uidKeywordFunc, // The file owner as a numeric value - "gid": gidKeywordFunc, // The file group as a numeric value - "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have - "uname": unameKeywordFunc, // The file owner as a symbolic name - "gname": gnameKeywordFunc, // The file group as a symbolic name - "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value - "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility - "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file - "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` - "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file - "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` - "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file - "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` - "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file - "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` - "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file - "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` - "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file - "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` + "size": sizeKeywordFunc, // The size, in bytes, of the file + "type": typeKeywordFunc, // The type of the file + "time": timeKeywordFunc, // The last modification time of the file + "link": linkKeywordFunc, // The target of the symbolic link when type=link + "uid": uidKeywordFunc, // The file owner as a numeric value + "gid": gidKeywordFunc, // The file group as a numeric value + "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have + "uname": unameKeywordFunc, // The file owner as a symbolic name + "gname": gnameKeywordFunc, // The file group as a symbolic name + "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value + "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility + "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file + "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5` + "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file + "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160` + "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file + "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1` + "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file + "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256` + "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file + "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384` + "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file + "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512` + "sha512256": hasherKeywordFunc("sha512digest", sha512.New512_256), // The SHA512/256 message digest of the file + "sha512256digest": hasherKeywordFunc("sha512digest", sha512.New512_256), // A synonym for `sha512256` "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword. diff --git a/vendor/github.com/vbatts/go-mtree/keywords.go b/vendor/github.com/vbatts/go-mtree/keywords.go index 46f9a8cf9..4e9c36190 100644 --- a/vendor/github.com/vbatts/go-mtree/keywords.go +++ b/vendor/github.com/vbatts/go-mtree/keywords.go @@ -316,6 +316,8 @@ func KeywordSynonym(name string) Keyword { retname = "sha384digest" case "sha512": retname = "sha512digest" + case "sha512256": + retname = "sha512256digest" case "xattrs": retname = "xattr" default: diff --git a/vendor/github.com/vbatts/go-mtree/lchtimes_unix.go b/vendor/github.com/vbatts/go-mtree/lchtimes_unix.go new file mode 100644 index 000000000..7cb5300b8 --- /dev/null +++ b/vendor/github.com/vbatts/go-mtree/lchtimes_unix.go @@ -0,0 +1,22 @@ +// +build darwin dragonfly freebsd openbsd linux netbsd solaris + +package mtree + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func lchtimes(name string, atime time.Time, mtime time.Time) error { + utimes := []unix.Timespec{ + unix.NsecToTimespec(atime.UnixNano()), + unix.NsecToTimespec(mtime.UnixNano()), + } + if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes, unix.AT_SYMLINK_NOFOLLOW); e != nil { + return &os.PathError{Op: "chtimes", Path: name, Err: e} + } + return nil + +} diff --git a/vendor/github.com/vbatts/go-mtree/lchtimes_unsupported.go b/vendor/github.com/vbatts/go-mtree/lchtimes_unsupported.go new file mode 100644 index 000000000..fac053256 --- /dev/null +++ b/vendor/github.com/vbatts/go-mtree/lchtimes_unsupported.go @@ -0,0 +1,11 @@ +// +build windows + +package mtree + +import ( + "time" +) + +func lchtimes(name string, atime time.Time, mtime time.Time) error { + return nil +} diff --git a/vendor/github.com/vbatts/go-mtree/tar.go b/vendor/github.com/vbatts/go-mtree/tar.go index e9599e650..51e251a06 100644 --- a/vendor/github.com/vbatts/go-mtree/tar.go +++ b/vendor/github.com/vbatts/go-mtree/tar.go @@ -9,7 +9,7 @@ import ( "path/filepath" "strings" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vbatts/go-mtree/pkg/govis" ) diff --git a/vendor/github.com/vbatts/go-mtree/update.go b/vendor/github.com/vbatts/go-mtree/update.go index bf0e7435a..5c37a1596 100644 --- a/vendor/github.com/vbatts/go-mtree/update.go +++ b/vendor/github.com/vbatts/go-mtree/update.go @@ -5,7 +5,7 @@ import ( "os" "sort" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) // DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk diff --git a/vendor/github.com/vbatts/go-mtree/updatefuncs.go b/vendor/github.com/vbatts/go-mtree/updatefuncs.go index 969d82d04..7bc2462f9 100644 --- a/vendor/github.com/vbatts/go-mtree/updatefuncs.go +++ b/vendor/github.com/vbatts/go-mtree/updatefuncs.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "github.com/vbatts/go-mtree/pkg/govis" ) diff --git a/vendor/github.com/vbatts/go-mtree/updatefuncs_linux.go b/vendor/github.com/vbatts/go-mtree/updatefuncs_linux.go index 4950c0f59..b7d7e834e 100644 --- a/vendor/github.com/vbatts/go-mtree/updatefuncs_linux.go +++ b/vendor/github.com/vbatts/go-mtree/updatefuncs_linux.go @@ -5,9 +5,6 @@ package mtree import ( "encoding/base64" "os" - "syscall" - "time" - "unsafe" "github.com/vbatts/go-mtree/xattr" ) @@ -22,37 +19,3 @@ func xattrUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) { } return os.Lstat(path) } - -func lchtimes(name string, atime time.Time, mtime time.Time) error { - var utimes [2]syscall.Timespec - utimes[0] = syscall.NsecToTimespec(atime.UnixNano()) - utimes[1] = syscall.NsecToTimespec(mtime.UnixNano()) - if e := utimensat(atFdCwd, name, (*[2]syscall.Timespec)(unsafe.Pointer(&utimes[0])), atSymlinkNofollow); e != nil { - return &os.PathError{Op: "chtimes", Path: name, Err: e} - } - return nil - -} - -// from uapi/linux/fcntl.h -// don't follow symlinks -const atSymlinkNofollow = 0x100 - -// special value for utimes as the FD for the current working directory -const atFdCwd = -0x64 - -func utimensat(dirfd int, path string, times *[2]syscall.Timespec, flags int) (err error) { - if len(times) != 2 { - return syscall.EINVAL - } - var _p0 *byte - _p0, err = syscall.BytePtrFromString(path) - if err != nil { - return - } - _, _, e1 := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(times)), uintptr(flags), 0, 0) - if e1 != 0 { - err = syscall.Errno(e1) - } - return -} diff --git a/vendor/github.com/vbatts/go-mtree/updatefuncs_unsupported.go b/vendor/github.com/vbatts/go-mtree/updatefuncs_unsupported.go index a55496499..9fc70e4be 100644 --- a/vendor/github.com/vbatts/go-mtree/updatefuncs_unsupported.go +++ b/vendor/github.com/vbatts/go-mtree/updatefuncs_unsupported.go @@ -4,13 +4,8 @@ package mtree import ( "os" - "time" ) func xattrUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) { return os.Lstat(path) } - -func lchtimes(name string, atime time.Time, mtime time.Time) error { - return nil -} diff --git a/vendor/github.com/vbatts/go-mtree/version.go b/vendor/github.com/vbatts/go-mtree/version.go index c6b79cd2e..ba089cb33 100644 --- a/vendor/github.com/vbatts/go-mtree/version.go +++ b/vendor/github.com/vbatts/go-mtree/version.go @@ -11,12 +11,12 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 0 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 4 + VersionMinor = 5 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 1 + VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "" + VersionDev = "-dev" ) // Version is the specification version that the package types support. diff --git a/vendor/github.com/opencontainers/image-tools/LICENSE b/vendor/github.com/wking/casengine/LICENSE similarity index 99% rename from vendor/github.com/opencontainers/image-tools/LICENSE rename to vendor/github.com/wking/casengine/LICENSE index 8dada3eda..d64569567 100644 --- a/vendor/github.com/opencontainers/image-tools/LICENSE +++ b/vendor/github.com/wking/casengine/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/github.com/wking/casengine/counter/counter.go b/vendor/github.com/wking/casengine/counter/counter.go new file mode 100644 index 000000000..de6636f26 --- /dev/null +++ b/vendor/github.com/wking/casengine/counter/counter.go @@ -0,0 +1,32 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package counter defines a byte-counting writer. One use case is measuring the size of content being streamed into CAS. +package counter + +type Counter struct { + count uint64 +} + +// Write implements io.Writer for Counter. +func (c *Counter) Write(p []byte) (n int, err error) { + c.count += uint64(len(p)) + return len(p), nil +} + +// Count returns the number of bytes which have been written to this +// Counter. +func (c *Counter) Count() (n uint64) { + return c.count +} diff --git a/vendor/github.com/wking/casengine/dir/digestlisterengine.go b/vendor/github.com/wking/casengine/dir/digestlisterengine.go new file mode 100644 index 000000000..3e0567b1f --- /dev/null +++ b/vendor/github.com/wking/casengine/dir/digestlisterengine.go @@ -0,0 +1,136 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dir + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/wking/casengine" + "golang.org/x/net/context" +) + +// GetDigest calculates the digest corresponding to a given relative +// path. This is effectively the inverse of URI Template expansion, +// and is required to support Digests. +type GetDigest func(path string) (digest digest.Digest, err error) + +// RegexpGetDigest is a helper structure for regular-expression based +// GetDigest implementations. +type RegexpGetDigest struct { + Regexp *regexp.Regexp +} + +// DigestListerEngine is a CAS engine based on the local filesystem. +type DigestListerEngine struct { + *Engine + + getDigest GetDigest +} + +// GetDigest implements GetDigest for RegexpGetDigest. +func (r *RegexpGetDigest) GetDigest(path string) (dig digest.Digest, err error) { + matches := make(map[string]string) + submatches := r.Regexp.FindStringSubmatch(path) + for i, submatchName := range r.Regexp.SubexpNames() { + if submatchName == "" { + continue + } + if i > len(submatches) { + return "", fmt.Errorf("%q does not match %q", path, r.Regexp.String()) + } + matches[submatchName] = submatches[i] + } + + algorithm, ok := matches["algorithm"] + if !ok { + return "", fmt.Errorf("no 'algorithm' capturing group in %q", r.Regexp.String()) + } + + encoded, ok := matches["encoded"] + if !ok { + return "", fmt.Errorf("no 'encoded' capturing group in %q", r.Regexp.String()) + } + + return digest.Parse(fmt.Sprintf("%s:%s", algorithm, encoded)) +} + +// NewDigestListerEngine creates a new CAS-engine instance that can +// list the digests it contains. Arguments are the same as for +// NewEngine, with an additional getDigest used to translate paths to +// digests. +func NewDigestListerEngine(ctx context.Context, path string, uri string, getDigest GetDigest) (engine casengine.DigestListerEngine, err error) { + base, err := NewEngine(ctx, path, uri) + if err != nil { + return nil, err + } + + return &DigestListerEngine{ + Engine: base.(*Engine), + getDigest: getDigest, + }, nil +} + +// Digests implements DigestLister.Digests. +func (engine *DigestListerEngine) Digests(ctx context.Context, algorithm digest.Algorithm, prefix string, size int, from int, callback casengine.DigestCallback) (err error) { + if size == 0 { + return nil + } + globAlgorithm := algorithm.String() + if globAlgorithm == "" { + globAlgorithm = "*" + } + globDigest := digest.Digest(fmt.Sprintf("%s:*", globAlgorithm)) + glob, err := engine.Engine.getPath(globDigest) + if err != nil { + return err + } + + matches, err := filepath.Glob(glob) + if err != nil { + return err + } + + offset := 0 + count := 0 + for _, match := range matches { + digest, err := engine.getDigest(match) + if err != nil { + logrus.Warnf("cannot compute digest for %q (%s)", match, err) + continue + } + + if algorithm.String() == "" || digest.Algorithm() == algorithm { + if prefix == "" || strings.HasPrefix(digest.Encoded(), prefix) { + if offset >= from { + err = callback(ctx, digest) + if err != nil { + return err + } + count++ + if size != -1 && count >= size { + return nil + } + } + offset++ + } + } + } + return nil +} diff --git a/vendor/github.com/wking/casengine/dir/engine.go b/vendor/github.com/wking/casengine/dir/engine.go new file mode 100644 index 000000000..52cec3546 --- /dev/null +++ b/vendor/github.com/wking/casengine/dir/engine.go @@ -0,0 +1,210 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dir implements a directory-based CAS engine. +package dir + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/wking/casengine" + "github.com/wking/casengine/read/template" + "golang.org/x/net/context" +) + +// Engine is a CAS engine based on the local filesystem. +type Engine struct { + temp string + reader *template.Engine + + // Algorithm selects the Algorithm used for Put. + Algorithm digest.Algorithm +} + +// NewEngine creates a new CAS-engine instance. The path argument is +// used as a base for expanding relative URIs and as a base for +// creating a temporary directory for storing partially-Put blobs. +// Moving the completed blob to its final location is more likely to +// be atomic if that temporary directory is on the same filesystem as +// the final location. +func NewEngine(ctx context.Context, path string, uri string) (engine casengine.Engine, err error) { + temp, err := ioutil.TempDir(path, ".casengine-") + if err != nil { + return nil, err + } + + base, err := url.Parse("file://" + path) + if err != nil { + return nil, err + } + + config := map[string]string{ + "uri": uri, + } + + reader, err := template.New(ctx, base, config) + if err != nil { + return nil, err + } + + readEngine, ok := reader.(*template.Engine) + if !ok { + return nil, fmt.Errorf("template.New() did not return a *template.Engine") + } + + if filepath.Separator != '/' { + return nil, fmt.Errorf("root path not implemented for filepath.Separator %q", filepath.Separator) + } + + readEngine.Client = &http.Client{ + Transport: http.NewFileTransport(http.Dir("/")), + } + + return &Engine{ + temp: temp, + reader: readEngine, + Algorithm: digest.SHA256, + }, nil +} + +// Get implements Reader.Get. +func (engine *Engine) Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) { + return engine.reader.Get(ctx, digest) +} + +// Algorithms implements AlgorithmLister.Algorithms. +func (engine *Engine) Algorithms(ctx context.Context, prefix string, size int, from int, callback casengine.AlgorithmCallback) (err error) { + if size == 0 { + return nil + } + offset := 0 + count := 0 + for _, algorithm := range []digest.Algorithm{ + digest.SHA256, + digest.SHA384, + digest.SHA512, + } { + if prefix == "" || strings.HasPrefix(algorithm.String(), prefix) { + if offset >= from { + err = callback(ctx, algorithm) + if err != nil { + return err + } + count++ + if size != -1 && count >= size { + return nil + } + } + offset++ + } + } + return nil +} + +// Put implements Writer.Put. +func (engine *Engine) Put(ctx context.Context, algorithm digest.Algorithm, reader io.Reader) (dig digest.Digest, err error) { + if algorithm.String() == "" { + algorithm = engine.Algorithm + } + digester := algorithm.Digester() + + file, err := ioutil.TempFile(engine.temp, "blob-") + if err != nil { + return "", err + } + + defer func() { + if err != nil { + err2 := os.Remove(file.Name()) + if err2 != nil { + logrus.Error(err2) + } + } + }() + + hashingWriter := io.MultiWriter(file, digester.Hash()) + _, err = io.Copy(hashingWriter, reader) + if err != nil { + return "", err + } + file.Close() + + dig = digester.Digest() + path, err := engine.getPath(dig) + if err != nil { + return "", err + } + + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return "", err + } + + err = os.Rename(file.Name(), path) + if err != nil { + return "", err + } + + return dig, nil +} + +// Delete implements Deleter.Delete. +func (engine *Engine) Delete(ctx context.Context, digest digest.Digest) (err error) { + path, err := engine.getPath(digest) + if err != nil { + return err + } + + err = os.Remove(path) + if os.IsNotExist(err) { + return nil + } + return err +} + +// Close implements Closer.Close. +func (engine *Engine) Close(ctx context.Context) (err error) { + err = os.RemoveAll(engine.temp) + if err != nil { + return err + } + + return engine.reader.Close(ctx) +} + +func (engine *Engine) getPath(digest digest.Digest) (path string, err error) { + if filepath.Separator != '/' { + return "", fmt.Errorf("getPath not implemented for filepath.Separator %q", filepath.Separator) + } + + uri, err := engine.reader.URI(digest) + if err != nil { + return "", err + } + + if uri.Scheme != "file" || uri.Opaque != "" || uri.User != nil || uri.Host != "" || uri.RawQuery != "" || uri.Fragment != "" { + return "", fmt.Errorf("invalid URI: %q", uri) + } + + return filepath.Join(uri.Path), nil +} diff --git a/vendor/github.com/wking/casengine/interface.go b/vendor/github.com/wking/casengine/interface.go new file mode 100644 index 000000000..a2098b4a2 --- /dev/null +++ b/vendor/github.com/wking/casengine/interface.go @@ -0,0 +1,217 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package casengine defines common interfaces for CAS engines. +package casengine + +import ( + "io" + + "github.com/opencontainers/go-digest" + "golang.org/x/net/context" +) + +// Reader represents a content-addressable storage engine reader. +type Reader interface { + + // Get returns a reader for retrieving a blob from the store. + // Returns os.ErrNotExist if the digest is not found. + // + // Implementations are *not* required to verify that the returned + // reader content matches the requested digest. Callers that need + // that verification are encouraged to use something like: + // + // rawReader, err := engine.Get(ctx, digest) + // defer rawReader.Close() + // verifier := digest.Verifier() + // verifiedReader := io.TeeReader(rawReader, verifier) + // consume(verifiedReader) + // if !verifier.Verified() { + // dieScreaming() + // } + // + // for streaming verification. + Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) +} + +// AlgorithmCallback templates an AlgorithmLister.Algorithms callback +// used for processing algorithms. AlgorithmLister.Algorithms for +// more details. +type AlgorithmCallback func(ctx context.Context, algorithm digest.Algorithm) (err error) + +// AlgorithmLister represents a content-addressable storage engine +// algorithm lister. +type AlgorithmLister interface { + + // Algorithms returns available algorithms from the store. The set + // of algorithms must include those which currently have stored + // digests, but may or may not include algorithms which may have stored + // digests in the future. + // + // Results are sorted alphabetically. + // + // Arguments: + // + // * ctx: gives callers a way to gracefully cancel a long-running + // list. + // * prefix: limits the result set to algorithms starting with that + // value. + // * size: limits the length of the result set to the first 'size' + // matches. A value of -1 means "all results". + // * from: shifts the result set to start from the 'from'th match. + // * callback: called for every matching algorithm. Algorithms + // returns any errors returned by callback and aborts further + // listing. + // + // For example, a store with digests like: + // + // * sha256:dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f + // * sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + // * sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e + // + // which also supports sha384 may have the following call/result pairs: + // + // * Algorithms(ctx, "", -1, 0, printAlgorithm) -> "sha256", "sha512" + // or + // Algorithms(ctx, "", -1, 0, printAlgorithm) -> "sha256", "sha384", "sha512" + // * Algorithms(ctx, "", 1, 0, printAlgorithm) -> "sha256" + // * Algorithms(ctx, "", 2, 1, printAlgorithm) -> "sha512" + // or + // Algorithms(ctx, "", 2, 1, printAlgorithm) -> "sha384", "sha512" + // * Algorithms(ctx, "sha5", -1, 0, printAlgorithm) -> "sha512" + Algorithms(ctx context.Context, prefix string, size int, from int, callback AlgorithmCallback) (err error) +} + +// DigestCallback templates an DigestLister.Digests callback used for +// processing algorithms. DigestLister.Digests for more details. +type DigestCallback func(ctx context.Context, digest digest.Digest) (err error) + +// DigestLister represents a content-addressable storage engine digest +// lister. +type DigestLister interface { + + // Digests returns available digests from the store. + // + // Results are sorted alphabetically. + // + // Arguments: + // + // * ctx: gives callers a way to gracefully cancel a long-running + // list. + // * algorithm: limits the result set to digests whose algorithm + // matches this value. An empty-string value means "all + // algorithms". + // * prefix: limits the result set to digests whose encoded part + // starts with that value. + // * size: limits the length of the result set to the first 'size' + // matches. A value of -1 means "all results". + // * from: shifts the result set to start from the 'from'th match. + // * callback: called for every matching digest. Digests returns + // any errors returned by callback and aborts further listing. + // + // For example, a store with digests like: + // + // * sha256:dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f + // * sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + // * sha512:374d794a95cdcfd8b35993185fef9ba368f160d8daf432d08ba9f1ed1e5abe6cc69291e0fa2fe0006a52570ef18c19def4e617c33ce52ef0a6e5fbe318cb0387 + // + // will have the following call/result pairs: + // + // * Digests(ctx, "", "", -1, 0, printDigest) -> + // "sha256:dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f" + // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + // "sha512:374d794a95cdcfd8b35993185fef9ba368f160d8daf432d08ba9f1ed1e5abe6cc69291e0fa2fe0006a52570ef18c19def4e617c33ce52ef0a6e5fbe318cb0387" + // * Digests(ctx, "sha256", "", -1, 0, printDigest) -> + // "sha256:dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f" + // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + // * Digests(ctx, "sha256", "e", -1, 0, printDigest) -> + // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + // * Digests(ctx, "", "", 2, 0, printDigest) -> + // "sha256:dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f" + // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + // * Digests(ctx, "", "", 2, 1, printDigest) -> + // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + // "sha512:374d794a95cdcfd8b35993185fef9ba368f160d8daf432d08ba9f1ed1e5abe6cc69291e0fa2fe0006a52570ef18c19def4e617c33ce52ef0a6e5fbe318cb0387" + Digests(ctx context.Context, algorithm digest.Algorithm, prefix string, size int, from int, callback DigestCallback) (err error) +} + +// Writer represents a content-addressable storage engine writer. +type Writer interface { + + // Put adds a new blob to the store. The action is idempotent; a + // nil return means "that content is stored at DIGEST" without + // implying "because of your Put()". + // + // The algorithm argument allows you to require a particular digest + // algorithm. Set to the empty string to allow the Writer to use + // its preferred algorithm. + Put(ctx context.Context, algorithm digest.Algorithm, reader io.Reader) (digest digest.Digest, err error) +} + +// Deleter represents a content-addressable storage engine deleter. +type Deleter interface { + + // Delete removes a blob from the store. The action is idempotent; a + // nil return means "that content is not in the store" without + // implying "because of your Delete()". + Delete(ctx context.Context, digest digest.Digest) (err error) +} + +// Closer represents a content-addressable storage engine closer. +type Closer interface { + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close(ctx context.Context) (err error) +} + +// ReadCloser is the interface that groups the basic Reader and Closer +// interfaces. +type ReadCloser interface { + Reader + Closer +} + +// ListDeleter is the interface that groups the basic AlgorithmLister, +// DigestLister, and Deleter interfaces. This combination is useful +// for garbage collection. +type ListDeleter interface { + AlgorithmLister + DigestLister + Deleter +} + +// WriteCloser is the interface that groups the basic Writer and +// Closer interfaces. +type WriteCloser interface { + Writer + Closer +} + +// Engine is the interface that groups all the basic interfaces +// defined in this package except for DigestLister. +type Engine interface { + Reader + AlgorithmLister + Writer + Deleter + Closer +} + +// DigestListerEngine is the interface that groups all the basic +// interfaces defined in this package. +type DigestListerEngine interface { + Engine + DigestLister +} diff --git a/vendor/github.com/wking/casengine/read/registry.go b/vendor/github.com/wking/casengine/read/registry.go new file mode 100644 index 000000000..4e3b1c24f --- /dev/null +++ b/vendor/github.com/wking/casengine/read/registry.go @@ -0,0 +1,30 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package read implements the CAS-engine protocol registry. +package read + +import ( + "net/url" + + "github.com/wking/casengine" + "golang.org/x/net/context" +) + +// New creates a new CAS-engine ReadCloser. +type New func(ctx context.Context, baseURI *url.URL, config interface{}) (engine casengine.ReadCloser, err error) + +// Constructors holds CAS-engine generators associated with registered +// protocol identifiers. +var Constructors = map[string]New{} diff --git a/vendor/github.com/wking/casengine/read/template/template.go b/vendor/github.com/wking/casengine/read/template/template.go new file mode 100644 index 000000000..3bed14493 --- /dev/null +++ b/vendor/github.com/wking/casengine/read/template/template.go @@ -0,0 +1,167 @@ +// Copyright 2017 casengine contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package template implements the OCI CAS Template Protocol v1. +// https://github.com/xiekeyang/oci-discovery/blob/0be7eae246ae9a975a76ca209c045043f0793572/cas-template.md +package template + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/jtacoma/uritemplates" + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/wking/casengine" + "github.com/wking/casengine/read" + "golang.org/x/net/context" +) + +// Engine implements the OCI CAS Template Protocol v1. +type Engine struct { + uri *uritemplates.UriTemplate + base *url.URL + + // Client allows callers to configure the HTTP client. Get will use + // http.DefaultClient if Client is not set. You can set this + // property with: + // + // engine, err := New(ctx, nil, config) + // // handle err and possibly engine.Close(ctx) + // engine.(*Engine).Client = yourCustomClient + Client *http.Client +} + +// New creates a new CAS-engine instance. +func New(ctx context.Context, baseURI *url.URL, config interface{}) (engine casengine.ReadCloser, err error) { + configMap, ok := config.(map[string]string) + if !ok { + configMap2, ok := config.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("CAS-template config is not a map[string]string: %v", config) + } + uriInterface, ok := configMap2["uri"] + if !ok { + return nil, fmt.Errorf("CAS-template config missing required 'uri' property: %v", configMap) + } + configMap = make(map[string]string) + configMap["uri"], ok = uriInterface.(string) + if !ok { + return nil, fmt.Errorf("CAS-template config 'uri' is not a string: %v", uriInterface) + } + } + + uriString, ok := configMap["uri"] + if !ok { + return nil, fmt.Errorf("CAS-template config missing required 'uri' property: %v", configMap) + } + + uriTemplate, err := uritemplates.Parse(uriString) + if err != nil { + return nil, err + } + + return &Engine{ + uri: uriTemplate, + base: baseURI, + }, nil +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *Engine) Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) { + request, err := engine.getPreFetch(digest) + if err != nil { + return nil, err + } + request = request.WithContext(ctx) + + client := engine.Client + if client == nil { + client = http.DefaultClient + } + logrus.Debugf("requesting %s from %s", digest, request.URL) + response, err := client.Do(request) + if err != nil { + return nil, err + } + + return engine.getPostFetch(response, digest) +} + +// Close releases resources held by the engine. +func (engine *Engine) Close(ctx context.Context) (err error) { + return nil +} + +// URI returns the expanded, resolved URI for digest. +func (engine *Engine) URI(digest digest.Digest) (uri *url.URL, err error) { + values := map[string]interface{}{ + "digest": string(digest), + "algorithm": string(digest.Algorithm()), + "encoded": digest.Encoded(), + } + + referenceURI, err := engine.uri.Expand(values) + if err != nil { + return nil, err + } + + parsedReference, err := url.Parse(referenceURI) + if err != nil { + return nil, err + } + + if !parsedReference.IsAbs() && engine.base == nil { + return nil, fmt.Errorf("cannot resolve relative %s without a base engine URI", parsedReference) + } + + return engine.base.ResolveReference(parsedReference), nil +} + +func (engine *Engine) getPreFetch(digest digest.Digest) (request *http.Request, err error) { + uri, err := engine.URI(digest) + if err != nil { + return nil, err + } + + return &http.Request{ + Method: "GET", + URL: uri, + }, nil +} + +func (engine *Engine) getPostFetch(response *http.Response, digest digest.Digest) (reader io.ReadCloser, err error) { + defer func() { + if err != nil { + response.Body.Close() + } + }() + + if response.StatusCode == http.StatusNotFound { + return nil, os.ErrNotExist + } + + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNoContent { + return nil, fmt.Errorf("requested %s but got %s", response.Request.URL, response.Status) + } + + return response.Body, nil +} + +func init() { + read.Constructors["oci-cas-template-v1"] = New +} diff --git a/vendor/github.com/xiekeyang/oci-discovery/LICENSE b/vendor/github.com/xiekeyang/oci-discovery/LICENSE new file mode 100644 index 000000000..a4a725b2e --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/LICENSE @@ -0,0 +1,13 @@ +Copyright 2017 oci-discovery contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/engine/config.go b/vendor/github.com/xiekeyang/oci-discovery/tools/engine/config.go new file mode 100644 index 000000000..769d09dea --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/engine/config.go @@ -0,0 +1,76 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package engine implements types common to both ref- and CAS-engines. +package engine + +import ( + "encoding/json" + "fmt" +) + +// Config represents a ref- or CAS-engine configuration. +type Config struct { + + // Protocol is a required part of refEngines and casEngines entries. + Protocol string `json:"protocol"` + + // Data holds the protocol-specific configuration. + Data map[string]interface{} +} + +// UnmarshalJSON reads a 'protocol' key into Protocol and any +// remaining keys into Data. +func (c *Config) UnmarshalJSON(b []byte) (err error) { + var dataInterface interface{} + if err := json.Unmarshal(b, &dataInterface); err != nil { + return err + } + + return c.unmarshalInterface(dataInterface) +} + +func (c *Config) unmarshalInterface(d interface{}) (err error) { + data, ok := d.(map[string]interface{}) + if !ok { + return fmt.Errorf("engine config is not a JSON object: %v", d) + } + + protocolInterface, ok := data["protocol"] + if !ok { + return fmt.Errorf("engine config missing required 'protocol' entry: %v", data) + } + + c.Protocol, ok = protocolInterface.(string) + if !ok { + return fmt.Errorf("engine config protocol is not a string: %v", protocolInterface) + } + + delete(data, "protocol") + c.Data = data + return nil +} + +// MarshalJSON writes any key/value pairs from Data and ensures +// 'protocol' is set equal to Protocol (clobbering a Data["protocol"] +// entry, if any). +func (c Config) MarshalJSON() ([]byte, error) { + var data map[string]interface{} + data = make(map[string]interface{}) + for key, value := range c.Data { + data[key] = value + } + data["protocol"] = c.Protocol + return json.Marshal(data) +} diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/engine/reference.go b/vendor/github.com/xiekeyang/oci-discovery/tools/engine/reference.go new file mode 100644 index 000000000..3bbed838c --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/engine/reference.go @@ -0,0 +1,89 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "encoding/json" + "fmt" + "net/url" +) + +// Reference holds a single resolved engine config. +type Reference struct { + + // Config is the engine configuration. + Config Config + + // URI is the source, if any, from which Config was retrieved. It + // can be used to expand any relative reference contained within + // Config. + URI *url.URL +} + +// UnmarshalJSON reads 'config' and 'uri' properties into Config and +// URI respectively. The main difference from the stock +// json.Unmarshal implementation is that the 'uri' value is read from +// a string instead of from an object with Scheme, Host, +// etc. properties. +func (reference *Reference) UnmarshalJSON(b []byte) (err error) { + var dataInterface interface{} + if err := json.Unmarshal(b, &dataInterface); err != nil { + return err + } + + data, ok := dataInterface.(map[string]interface{}) + if !ok { + return fmt.Errorf("reference is not a JSON object: %v", dataInterface) + } + + configInterface, ok := data["config"] + if !ok { + return fmt.Errorf("reference missing required 'config' entry: %v", data) + } + + err = reference.Config.unmarshalInterface(configInterface) + if err != nil { + return err + } + + uriInterface, ok := data["uri"] + if !ok { + reference.URI = nil + } else { + uriString, ok := uriInterface.(string) + if !ok { + return fmt.Errorf("reference uri is not a string: %v", uriInterface) + } + reference.URI, err = url.Parse(uriString) + if err != nil { + return err + } + } + + return nil +} + +// MarshalJSON writes 'config' and 'uri' properties to the output +// object. The main difference from the stock json.Marshal +// implementation is that the 'uri' value is written as a string instead +// of an object with Scheme, Host, etc. properties. +func (reference Reference) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{} + data["config"] = reference.Config + if reference.URI != nil { + data["uri"] = reference.URI.String() + } + return json.Marshal(data) +} diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/interface.go b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/interface.go new file mode 100644 index 000000000..767d99d97 --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/interface.go @@ -0,0 +1,37 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package refengine + +import ( + "net/url" + + "golang.org/x/net/context" +) + +// Engine represents a reference engine. +type Engine interface { + + // Get returns an array of potential Merkle roots from the store. + // When no results are found, roots will be an empty array and err + // will be nil. + Get(ctx context.Context, name string) (roots []MerkleRoot, err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close(ctx context.Context) (err error) +} + +// New creates a new ref-engine instance. +type New func(ctx context.Context, baseURI *url.URL, config interface{}) (engine Engine, err error) diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/registry.go b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/registry.go new file mode 100644 index 000000000..515868596 --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/registry.go @@ -0,0 +1,21 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package refengine implements the ref-engine protocol registry and +// defines the common interfaces. +package refengine + +// Constructors holds ref-engine generators associated with registered +// protocol identifiers. +var Constructors = map[string]New{} diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/root.go b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/root.go new file mode 100644 index 000000000..dec007943 --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/refengine/root.go @@ -0,0 +1,99 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package refengine + +import ( + "encoding/json" + "fmt" + "net/url" +) + +// MerkleRoot holds a single resolved Merkle root. +type MerkleRoot struct { + // MediaType is the media type of Root. + MediaType string + + // Root is the Merkle root object. While this may be of any type. + // OCI tools will generally use image-spec Descriptors. + Root interface{} + + // URI is the source, if any, from which Root was retrieved. It can + // be used to expand any relative reference contained within Root. + URI *url.URL +} + +// UnmarshalJSON reads 'mediaType', 'root', and 'uri' properties into +// MediaType, Root, and URI respectively. The main difference from +// the stock json.Unmarshal implementation is that the 'uri' value is +// read from a string instead of from an object with Scheme, Host, +// etc. properties. +func (root *MerkleRoot) UnmarshalJSON(b []byte) (err error) { + var dataInterface interface{} + if err := json.Unmarshal(b, &dataInterface); err != nil { + return err + } + + data, ok := dataInterface.(map[string]interface{}) + if !ok { + return fmt.Errorf("merkle root is not a JSON object: %v", dataInterface) + } + + mediaTypeInterface, ok := data["mediaType"] + if !ok { + root.MediaType = "" + } else { + mediaTypeString, ok := mediaTypeInterface.(string) + if !ok { + return fmt.Errorf("merkle root mediaType is not a string: %v", mediaTypeInterface) + } + root.MediaType = mediaTypeString + } + + root.Root = data["root"] + + uriInterface, ok := data["uri"] + if !ok { + root.URI = nil + } else { + uriString, ok := uriInterface.(string) + if !ok { + return fmt.Errorf("merkle root uri is not a string: %v", uriInterface) + } + root.URI, err = url.Parse(uriString) + if err != nil { + return err + } + } + + return nil +} + +// MarshalJSON writes 'mediaType', 'root', and 'uri' properties to the +// output object. The main difference from the stock json.Marshal +// implementation is that the 'uri' value is written as a string +// instead of an object with Scheme, Host, etc. properties. +func (root MerkleRoot) MarshalJSON() ([]byte, error) { + data := map[string]interface{}{} + if root.MediaType != "" { + data["mediaType"] = root.MediaType + } + if root.Root != nil { + data["root"] = root.Root + } + if root.URI != nil { + data["uri"] = root.URI.String() + } + return json.Marshal(data) +} diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/engines.go b/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/engines.go new file mode 100644 index 000000000..08afba04b --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/engines.go @@ -0,0 +1,74 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package refenginediscovery + +import ( + "github.com/sirupsen/logrus" + "github.com/xiekeyang/oci-discovery/tools/engine" + "github.com/xiekeyang/oci-discovery/tools/refengine" + "golang.org/x/net/context" +) + +// ResolvedNameCallback templates a callback for use in ResolveName. +type ResolvedNameCallback func(ctx context.Context, root refengine.MerkleRoot, casEngines []engine.Reference) (err error) + +// ResolveName iterates over engines calling Engine.RefEngines to get +// potential ref-engine configs. Then it iterates over those +// ref-engine configs, instantiates a ref engine, and queries that ref +// engine for matching Merkle roots, calling resolvedNameCallback on +// each one. ResolveName returns any errors returned by +// resolvedNameCallback and aborts further iteration. Other errors +// (e.g. in initializing a ref engine) generate logged warnings but +// are otherwise ignored. +func ResolveName(ctx context.Context, engines []Engine, name string, resolvedNameCallback ResolvedNameCallback) (err error) { + for _, engine := range engines { + err = engine.RefEngines( + ctx, + name, + func(ctx context.Context, refEngine RefEngineReference) (err error) { + constructor, ok := refengine.Constructors[refEngine.Config.Config.Protocol] + if !ok { + logrus.Debugf("unsupported ref-engine protocol %q (%v)", refEngine.Config.Config.Protocol, refengine.Constructors) + return nil + } + eng, err := constructor(ctx, refEngine.Config.URI, refEngine.Config.Config.Data) + if err != nil { + logrus.Warnf("failed to initialize %s ref engine with %v: %s", refEngine.Config.Config.Protocol, refEngine.Config.Config.Data, err) + return nil + } + defer eng.Close(ctx) + + roots, err := eng.Get(ctx, name) + if err != nil { + logrus.Warn(err) + return nil + } + for _, root := range roots { + err = resolvedNameCallback(ctx, root, refEngine.CASEngines) + if err != nil { + return err + } + } + + return nil + }, + ) + if err != nil { + return err + } + } + + return nil +} diff --git a/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/interface.go b/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/interface.go new file mode 100644 index 000000000..79d2311fd --- /dev/null +++ b/vendor/github.com/xiekeyang/oci-discovery/tools/refenginediscovery/interface.go @@ -0,0 +1,70 @@ +// Copyright 2017 oci-discovery contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package refenginediscovery + +import ( + "net/url" + + "github.com/xiekeyang/oci-discovery/tools/engine" + "golang.org/x/net/context" +) + +// Engines holds application/vnd.oci.ref-engines.v1+json data. +type Engines struct { + + // RefEngines is an array of ref-engine configurations. + RefEngines []engine.Config `json:"refEngines,omitempty"` + + // CASEngines is an array of CAS-engine configurations. + CASEngines []engine.Config `json:"casEngines,omitempty"` +} + +// RefEnginesReference holds resolved Engines data. +type RefEnginesReference struct { + + // Engines holds the resolved Engines declaration. + Engines Engines + + // URI is the source, if any, from which Engines was retrieved. It + // can be used to expand any relative reference contained within + // Engines. + URI *url.URL +} + +// RefEngineReference holds a single resolved ref-engine object. +type RefEngineReference struct { + // Config holds a single resolved ref-engine config. + Config engine.Reference + + // CASEngines holds the ref-engines object's CAS-engine suggestions, + // if any. + CASEngines []engine.Reference +} + +// RefEngineReferenceCallback templates a callback for use in RefEngineReferences. +type RefEngineReferenceCallback func(ctx context.Context, reference RefEngineReference) (err error) + +// Engine represents a ref-engine discovery engine. +type Engine interface { + + // RefEngines calculates ref engines using Ref-Engine Discovery and + // calls refEngineCallback on each one. Discover returns any errors + // returned by refEngineCallback and aborts further iteration. + RefEngines(ctx context.Context, name string, callback RefEngineReferenceCallback) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close(ctx context.Context) (err error) +}