Documentation
¶
Overview ¶
Package blob provides a file archive format optimized for random access via HTTP range requests against OCI registries.
This package provides a unified high-level API through Client for pushing and pulling blob archives to/from OCI registries. For low-level archive operations without registry interaction, use the [core] subpackage.
Archives consist of two OCI blobs:
- Index blob: FlatBuffers-encoded file metadata enabling O(log n) lookups
- Data blob: Concatenated file contents, sorted by path for efficient directory fetches
Quick Start ¶
Push a directory to a registry:
c, err := blob.NewClient(blob.WithDockerConfig())
if err != nil {
return err
}
err = c.Push(ctx, "ghcr.io/myorg/myarchive:v1", "./src",
blob.PushWithCompression(blob.CompressionZstd),
)
Pull and read files:
archive, err := c.Pull(ctx, "ghcr.io/myorg/myarchive:v1")
if err != nil {
return err
}
content, err := archive.ReadFile("config.json")
Caching ¶
Use WithCacheDir for automatic caching of all blob metadata and content:
c, err := blob.NewClient(
blob.WithDockerConfig(),
blob.WithCacheDir("/var/cache/blob"),
)
For fine-grained control, use individual cache options like WithRefCacheDir, WithManifestCacheDir, WithContentCacheDir, etc.
Policies ¶
Add verification policies to enforce security requirements on pull:
sigPolicy, _ := sigstore.NewPolicy(sigstore.WithIdentity(issuer, subject))
c, err := blob.NewClient(
blob.WithDockerConfig(),
blob.WithPolicy(sigPolicy),
)
Index ¶
- Constants
- Variables
- type Archive
- type ByteSource
- type ChangeDetection
- type Client
- func (c *Client) Fetch(ctx context.Context, ref string, opts ...FetchOption) (*Manifest, error)
- func (c *Client) Inspect(ctx context.Context, ref string, opts ...InspectOption) (*InspectResult, error)
- func (c *Client) Pull(ctx context.Context, ref string, opts ...PullOption) (*Archive, error)
- func (c *Client) Push(ctx context.Context, ref, srcDir string, opts ...PushOption) error
- func (c *Client) PushArchive(ctx context.Context, ref string, archive *blobcore.Blob, opts ...PushOption) error
- func (c *Client) Sign(ctx context.Context, ref string, signer ManifestSigner, opts ...SignOption) (string, error)
- func (c *Client) Tag(ctx context.Context, ref, digest string) error
- type Compression
- type CopyOption
- type Entry
- type EntryView
- type FetchOption
- type IndexView
- type InspectOption
- type InspectResult
- func (r *InspectResult) CompressionRatio() float64
- func (r *InspectResult) Created() time.Time
- func (r *InspectResult) DataBlobSize() int64
- func (r *InspectResult) Digest() string
- func (r *InspectResult) FileCount() int
- func (r *InspectResult) Index() *blobcore.IndexView
- func (r *InspectResult) IndexBlobSize() int64
- func (r *InspectResult) Manifest() *registry.BlobManifest
- func (r *InspectResult) Referrers(ctx context.Context, artifactType string) ([]Referrer, error)
- func (r *InspectResult) TotalCompressedSize() uint64
- func (r *InspectResult) TotalUncompressedSize() uint64
- type Manifest
- type ManifestSigner
- type Option
- func WithAnonymous() Option
- func WithBlockCache(cache corecache.BlockCache) Option
- func WithBlockCacheDir(dir string) Option
- func WithCacheDir(dir string) Option
- func WithContentCache(cache corecache.Cache) Option
- func WithContentCacheDir(dir string) Option
- func WithDockerConfig() Option
- func WithIndexCache(cache registrycache.IndexCache) Option
- func WithIndexCacheDir(dir string) Option
- func WithManifestCache(cache registrycache.ManifestCache) Option
- func WithManifestCacheDir(dir string) Option
- func WithPlainHTTP(enabled bool) Option
- func WithPolicies(policies ...Policy) Option
- func WithPolicy(policy Policy) Option
- func WithRefCache(cache registrycache.RefCache) Option
- func WithRefCacheDir(dir string) Option
- func WithRefCacheTTL(ttl time.Duration) Option
- func WithStaticCredentials(registry, username, password string) Option
- func WithStaticToken(registry, token string) Option
- func WithUserAgent(ua string) Option
- type Policy
- type PolicyClient
- type PolicyRequest
- type PullOption
- type PushOption
- func PushWithAnnotations(annotations map[string]string) PushOption
- func PushWithChangeDetection(cd ChangeDetection) PushOption
- func PushWithCompression(c Compression) PushOption
- func PushWithMaxFiles(n int) PushOption
- func PushWithSkipCompression(fns ...SkipCompressionFunc) PushOption
- func PushWithTags(tags ...string) PushOption
- type Referrer
- type SignOption
- type SkipCompressionFunc
Constants ¶
const ( DefaultContentCacheSize int64 = 100 << 20 // 100 MB DefaultBlockCacheSize int64 = 50 << 20 // 50 MB DefaultIndexCacheSize int64 = 50 << 20 // 50 MB DefaultManifestCacheSize int64 = 10 << 20 // 10 MB DefaultRefCacheSize int64 = 5 << 20 // 5 MB DefaultRefCacheTTL = 5 * time.Minute )
Default cache size limits for WithCacheDir.
const ( CompressionNone = blobcore.CompressionNone CompressionZstd = blobcore.CompressionZstd )
Compression constants.
const ( ChangeDetectionNone = blobcore.ChangeDetectionNone ChangeDetectionStrict = blobcore.ChangeDetectionStrict )
ChangeDetection constants.
const ( DefaultIndexName = blobcore.DefaultIndexName DefaultDataName = blobcore.DefaultDataName )
Default file names for blob archives.
const DefaultMaxFiles = blobcore.DefaultMaxFiles
DefaultMaxFiles is the default limit used when no MaxFiles option is set.
Variables ¶
var ( // ErrHashMismatch is returned when file content hash doesn't match the expected hash. ErrHashMismatch = blobcore.ErrHashMismatch // ErrDecompression is returned when decompression fails. ErrDecompression = blobcore.ErrDecompression // ErrSizeOverflow is returned when a size value overflows. ErrSizeOverflow = blobcore.ErrSizeOverflow // ErrSymlink is returned when a symlink is encountered during archive creation. ErrSymlink = blobcore.ErrSymlink // ErrTooManyFiles is returned when the archive contains more files than allowed. ErrTooManyFiles = blobcore.ErrTooManyFiles )
Errors re-exported from core.
var ( // ErrNotFound is returned when a blob archive does not exist at the reference. ErrNotFound = registry.ErrNotFound // ErrInvalidReference is returned when a reference string is malformed. ErrInvalidReference = registry.ErrInvalidReference // ErrInvalidManifest is returned when a manifest is not a valid blob archive manifest. ErrInvalidManifest = registry.ErrInvalidManifest // ErrMissingIndex is returned when the manifest does not contain an index blob. ErrMissingIndex = registry.ErrMissingIndex // ErrMissingData is returned when the manifest does not contain a data blob. ErrMissingData = registry.ErrMissingData // ErrDigestMismatch is returned when content does not match its expected digest. ErrDigestMismatch = registry.ErrDigestMismatch // ErrPolicyViolation is returned when a policy rejects a manifest. ErrPolicyViolation = registry.ErrPolicyViolation // ErrReferrersUnsupported is returned when referrers are not supported by the OCI client. ErrReferrersUnsupported = registry.ErrReferrersUnsupported )
Errors re-exported from registry.
var ( CopyWithOverwrite = blobcore.CopyWithOverwrite CopyWithPreserveMode = blobcore.CopyWithPreserveMode CopyWithPreserveTimes = blobcore.CopyWithPreserveTimes CopyWithCleanDest = blobcore.CopyWithCleanDest CopyWithWorkers = blobcore.CopyWithWorkers CopyWithReadConcurrency = blobcore.CopyWithReadConcurrency CopyWithReadAheadBytes = blobcore.CopyWithReadAheadBytes )
Copy options re-exported from core.
var DefaultSkipCompression = blobcore.DefaultSkipCompression
DefaultSkipCompression returns a SkipCompressionFunc that skips small files and known already-compressed extensions.
Functions ¶
This section is empty.
Types ¶
type Archive ¶
Archive wraps a pulled blob archive with integrated caching. It embeds *core.Blob, so all Blob methods are directly accessible.
type ByteSource ¶
type ByteSource = blobcore.ByteSource
ByteSource provides random access to the data blob.
type ChangeDetection ¶
type ChangeDetection = blobcore.ChangeDetection
ChangeDetection controls how strictly file changes are detected during creation.
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client provides high-level operations for blob archives in OCI registries.
Client wraps a registry client and adds blob-archive-specific functionality including automatic archive creation, content caching, and policy enforcement.
func NewClient ¶
NewClient creates a new blob archive client with the given options.
If no authentication is configured, anonymous access is used. Use WithDockerConfig to read credentials from ~/.docker/config.json.
func (*Client) Fetch ¶
Fetch retrieves the manifest for a blob archive without downloading data.
This is useful for inspecting archive metadata or checking if an archive exists without the overhead of downloading blob content.
func (*Client) Inspect ¶
func (c *Client) Inspect(ctx context.Context, ref string, opts ...InspectOption) (*InspectResult, error)
Inspect retrieves archive metadata without downloading file data.
This fetches the manifest and index blob, providing access to:
- Manifest metadata (digest, created time, annotations)
- File index (paths, sizes, hashes, modes)
- Archive statistics (file count, total size)
The data blob is NOT downloaded. Use Client.Pull to extract files.
Referrers (signatures, attestations) can be fetched on-demand via InspectResult.Referrers.
func (*Client) Pull ¶
Pull retrieves an archive from the registry with lazy data loading.
The returned Archive wraps a core.Blob with caching support. File data is fetched on demand via HTTP range requests.
Use PullWithSkipCache to bypass all caches and force a fresh fetch.
func (*Client) Push ¶
Push creates an archive from srcDir and pushes it to the registry.
This is the most common operation - create + push in one call. The ref must include a tag (e.g., "registry.com/repo:v1.0.0").
Use PushWithTags to apply additional tags to the same manifest. Use PushWithCompression to configure compression (default: none).
func (*Client) PushArchive ¶
func (c *Client) PushArchive(ctx context.Context, ref string, archive *blobcore.Blob, opts ...PushOption) error
PushArchive pushes an existing archive to the registry.
Use when you have a pre-created archive (e.g., from blobcore.CreateBlob). The ref must include a tag (e.g., "registry.com/repo:v1.0.0").
Use PushWithTags to apply additional tags to the same manifest.
func (*Client) Sign ¶
func (c *Client) Sign(ctx context.Context, ref string, signer ManifestSigner, opts ...SignOption) (string, error)
Sign creates a sigstore signature for a manifest and attaches it as a referrer.
The ref must include a tag or digest. The signer creates the signature bundle, which is pushed as an OCI 1.1 referrer artifact linked to the manifest.
Returns the digest of the signature manifest.
type Compression ¶
type Compression = blobcore.Compression
Compression identifies the compression algorithm used for a file.
type CopyOption ¶
type CopyOption = blobcore.CopyOption
CopyOption configures CopyTo and CopyDir operations.
type FetchOption ¶
type FetchOption func(*fetchConfig)
FetchOption configures a Fetch operation.
func FetchWithSkipCache ¶
func FetchWithSkipCache() FetchOption
FetchWithSkipCache bypasses the ref and manifest caches for this fetch.
The fetched manifest is still added to the cache after retrieval.
type InspectOption ¶
type InspectOption func(*inspectConfig)
InspectOption configures an Inspect operation.
func InspectWithMaxIndexSize ¶
func InspectWithMaxIndexSize(size int64) InspectOption
InspectWithMaxIndexSize sets the maximum index size to fetch.
func InspectWithSkipCache ¶
func InspectWithSkipCache() InspectOption
InspectWithSkipCache bypasses all caches for this inspection.
type InspectResult ¶
type InspectResult struct {
// contains filtered or unexported fields
}
InspectResult contains metadata about a blob archive without the data blob.
It provides access to the manifest, file index, and optionally referrers, enabling inspection of archive contents without downloading file data.
func (*InspectResult) CompressionRatio ¶
func (r *InspectResult) CompressionRatio() float64
CompressionRatio returns the ratio of compressed to uncompressed size. Returns 1.0 if the archive is uncompressed or has no files. This requires iterating all entries on first call; the result is cached.
func (*InspectResult) Created ¶
func (r *InspectResult) Created() time.Time
Created returns the archive creation time.
func (*InspectResult) DataBlobSize ¶
func (r *InspectResult) DataBlobSize() int64
DataBlobSize returns the size of the data blob (compressed archive).
func (*InspectResult) Digest ¶
func (r *InspectResult) Digest() string
Digest returns the manifest digest.
func (*InspectResult) FileCount ¶
func (r *InspectResult) FileCount() int
FileCount returns the number of files in the archive.
func (*InspectResult) Index ¶
func (r *InspectResult) Index() *blobcore.IndexView
Index returns the file index view.
func (*InspectResult) IndexBlobSize ¶
func (r *InspectResult) IndexBlobSize() int64
IndexBlobSize returns the size of the index blob.
func (*InspectResult) Manifest ¶
func (r *InspectResult) Manifest() *registry.BlobManifest
Manifest returns the OCI manifest metadata.
func (*InspectResult) Referrers ¶
Referrers fetches referrer artifacts (signatures, attestations, etc.).
The artifactType parameter filters referrers by type. Pass "" to get all. Returns registry.ErrReferrersUnsupported if the registry doesn't support referrers.
func (*InspectResult) TotalCompressedSize ¶
func (r *InspectResult) TotalCompressedSize() uint64
TotalCompressedSize returns the sum of all compressed file sizes in the data blob. This requires iterating all entries on first call; the result is cached.
func (*InspectResult) TotalUncompressedSize ¶
func (r *InspectResult) TotalUncompressedSize() uint64
TotalUncompressedSize returns the sum of all uncompressed file sizes. This requires iterating all entries on first call; the result is cached.
type Manifest ¶
type Manifest = registry.BlobManifest
Manifest represents a blob archive manifest from an OCI registry.
type ManifestSigner ¶
type ManifestSigner = registry.ManifestSigner
ManifestSigner signs OCI manifest payloads.
type Option ¶
Option configures a Client.
func WithAnonymous ¶
func WithAnonymous() Option
WithAnonymous forces anonymous access, ignoring any configured credentials.
func WithBlockCache ¶
func WithBlockCache(cache corecache.BlockCache) Option
WithBlockCache sets a custom block cache implementation. Import github.com/meigma/blob/core/cache/disk for the disk implementation.
func WithBlockCacheDir ¶
WithBlockCacheDir enables HTTP range block caching in the specified directory with the default size limit (DefaultBlockCacheSize).
func WithCacheDir ¶
WithCacheDir enables all caches with default sizes in subdirectories of dir.
This creates:
- dir/content/ - file content cache (100 MB)
- dir/blocks/ - HTTP range block cache (50 MB)
- dir/refs/ - tag→digest cache (5 MB)
- dir/manifests/ - manifest cache (10 MB)
- dir/indexes/ - index blob cache (50 MB)
For custom sizes or selective caching, use individual cache options.
func WithContentCache ¶
WithContentCache sets a custom content cache implementation. Import github.com/meigma/blob/core/cache/disk for the disk implementation.
func WithContentCacheDir ¶
WithContentCacheDir enables file content caching in the specified directory with the default size limit (DefaultContentCacheSize).
func WithDockerConfig ¶
func WithDockerConfig() Option
WithDockerConfig enables reading credentials from ~/.docker/config.json. This is the recommended way to authenticate with registries.
func WithIndexCache ¶
func WithIndexCache(cache registrycache.IndexCache) Option
WithIndexCache sets a custom index cache implementation. Import github.com/meigma/blob/registry/cache/disk for the disk implementation.
func WithIndexCacheDir ¶
WithIndexCacheDir enables index blob caching in the specified directory with the default size limit (DefaultIndexCacheSize).
func WithManifestCache ¶
func WithManifestCache(cache registrycache.ManifestCache) Option
WithManifestCache sets a custom manifest cache implementation. Import github.com/meigma/blob/registry/cache/disk for the disk implementation.
func WithManifestCacheDir ¶
WithManifestCacheDir enables manifest caching in the specified directory with the default size limit (DefaultManifestCacheSize).
func WithPlainHTTP ¶
WithPlainHTTP enables plain HTTP (no TLS) for registries. This is useful for local development registries.
func WithPolicies ¶
WithPolicies adds multiple policies that must pass for Fetch and Pull operations.
func WithPolicy ¶
WithPolicy adds a policy that must pass for Fetch and Pull operations. Policies are evaluated in order; the first failure stops evaluation.
func WithRefCache ¶
func WithRefCache(cache registrycache.RefCache) Option
WithRefCache sets a custom reference cache implementation. Import github.com/meigma/blob/registry/cache/disk for the disk implementation.
func WithRefCacheDir ¶
WithRefCacheDir enables tag-to-digest reference caching in the specified directory with the default size limit (DefaultRefCacheSize).
func WithRefCacheTTL ¶
WithRefCacheTTL sets the TTL for reference cache entries. This determines how long tag→digest mappings are considered fresh.
func WithStaticCredentials ¶
WithStaticCredentials sets static username/password credentials for a registry. The registry parameter should be the registry host (e.g., "ghcr.io").
func WithStaticToken ¶
WithStaticToken sets a static bearer token for a registry. The registry parameter should be the registry host (e.g., "ghcr.io").
func WithUserAgent ¶
WithUserAgent sets the User-Agent header for registry requests.
type Policy ¶
Policy evaluates whether a manifest is trusted.
Policies are evaluated during Fetch and Pull operations. If any policy returns an error, the operation fails with ErrPolicyViolation.
type PolicyClient ¶
type PolicyClient = registry.PolicyClient
PolicyClient exposes minimal client capabilities for policies.
type PolicyRequest ¶
type PolicyRequest = registry.PolicyRequest
PolicyRequest provides context for policy evaluation.
type PullOption ¶
type PullOption func(*pullConfig)
PullOption configures a Pull operation.
func PullWithDecoderConcurrency ¶
func PullWithDecoderConcurrency(n int) PullOption
PullWithDecoderConcurrency sets the zstd decoder concurrency (default: 1). Values < 0 are treated as 0 (use GOMAXPROCS).
func PullWithDecoderLowmem ¶
func PullWithDecoderLowmem(enabled bool) PullOption
PullWithDecoderLowmem sets whether the zstd decoder should use low-memory mode (default: false).
func PullWithMaxFileSize ¶
func PullWithMaxFileSize(limit uint64) PullOption
PullWithMaxFileSize limits the maximum per-file size (compressed and uncompressed). Set limit to 0 to disable the limit.
func PullWithMaxIndexSize ¶
func PullWithMaxIndexSize(maxBytes int64) PullOption
PullWithMaxIndexSize sets the maximum number of bytes allowed for the index blob.
Use a value <= 0 to disable the limit.
func PullWithSkipCache ¶
func PullWithSkipCache() PullOption
PullWithSkipCache bypasses the ref and manifest caches.
This forces a fresh fetch from the registry even if cached data exists.
func PullWithVerifyOnClose ¶
func PullWithVerifyOnClose(enabled bool) PullOption
PullWithVerifyOnClose controls whether Close drains the file to verify the hash.
When false, Close returns without reading the remaining data. Integrity is only guaranteed when callers read to EOF.
type PushOption ¶
type PushOption func(*pushConfig)
PushOption configures a Push or PushArchive operation.
func PushWithAnnotations ¶
func PushWithAnnotations(annotations map[string]string) PushOption
PushWithAnnotations sets custom annotations on the manifest.
Standard annotations like org.opencontainers.image.created are set automatically and can be overridden.
func PushWithChangeDetection ¶
func PushWithChangeDetection(cd ChangeDetection) PushOption
PushWithChangeDetection controls whether the writer verifies files did not change during archive creation.
func PushWithCompression ¶
func PushWithCompression(c Compression) PushOption
PushWithCompression sets the compression algorithm for archive creation. Use CompressionNone to store files uncompressed, CompressionZstd for zstd.
func PushWithMaxFiles ¶
func PushWithMaxFiles(n int) PushOption
PushWithMaxFiles limits the number of files included in the archive. Zero uses DefaultMaxFiles. Negative means no limit.
func PushWithSkipCompression ¶
func PushWithSkipCompression(fns ...SkipCompressionFunc) PushOption
PushWithSkipCompression adds predicates that decide to store a file uncompressed. If any predicate returns true, compression is skipped for that file.
func PushWithTags ¶
func PushWithTags(tags ...string) PushOption
PushWithTags applies additional tags to the pushed manifest.
The primary tag from the ref is always applied. These tags are applied after the initial push succeeds.
type Referrer ¶
type Referrer struct {
// Digest is the content-addressable identifier (e.g., "sha256:abc123...").
Digest string
// Size is the size of the referrer content in bytes.
Size int64
// MediaType identifies the format of the referrer content.
MediaType string
// ArtifactType identifies the type of artifact (e.g., signature, attestation).
ArtifactType string
// Annotations contains optional metadata key-value pairs.
Annotations map[string]string
}
Referrer describes an artifact that references a manifest.
Common referrer types include:
- Sigstore signatures (application/vnd.dev.sigstore.bundle.v0.3+json)
- In-toto attestations (application/vnd.in-toto+json)
- SLSA provenance documents
type SkipCompressionFunc ¶
type SkipCompressionFunc = blobcore.SkipCompressionFunc
SkipCompressionFunc returns true when a file should be stored uncompressed.
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
client
module
|
|
|
Package blob provides a file archive format optimized for random access via HTTP range requests against OCI registries.
|
Package blob provides a file archive format optimized for random access via HTTP range requests against OCI registries. |
|
cache
Package cache provides content-addressed caching for blob archives.
|
Package cache provides content-addressed caching for blob archives. |
|
cache/disk
Package disk provides a disk-backed cache implementation.
|
Package disk provides a disk-backed cache implementation. |
|
http
Package http provides a ByteSource backed by HTTP range requests.
|
Package http provides a ByteSource backed by HTTP range requests. |
|
internal/batch
Package batch provides batch processing for reading multiple entries from a blob archive.
|
Package batch provides batch processing for reading multiple entries from a blob archive. |
|
internal/blobtype
Package blobtype defines shared types used across the blob package and its internal packages.
|
Package blobtype defines shared types used across the blob package and its internal packages. |
|
internal/fb
Package fb contains FlatBuffers-generated code for the blob index format.
|
Package fb contains FlatBuffers-generated code for the blob index format. |
|
internal/file
Package file provides internal file reading operations for the blob package.
|
Package file provides internal file reading operations for the blob package. |
|
internal/index
Package index provides FlatBuffers index loading and lookup for blob archives.
|
Package index provides FlatBuffers index loading and lookup for blob archives. |
|
internal/platform
Package platform provides platform-specific file operations.
|
Package platform provides platform-specific file operations. |
|
internal/sizing
Package sizing provides safe size arithmetic and conversions to prevent overflow.
|
Package sizing provides safe size arithmetic and conversions to prevent overflow. |
|
internal/write
Package write provides internal file writing operations for the blob package.
|
Package write provides internal file writing operations for the blob package. |
|
testutil
Package testutil provides test utilities for the blob package.
|
Package testutil provides test utilities for the blob package. |
|
Package policy provides composition utilities for combining multiple policies.
|
Package policy provides composition utilities for combining multiple policies. |
|
sigstore
module
|
|
|
Package registry provides a high-level client for pushing and pulling blob archives to/from OCI registries.
|
Package registry provides a high-level client for pushing and pulling blob archives to/from OCI registries. |
|
cache
Package cache provides caching interfaces for the blob client.
|
Package cache provides caching interfaces for the blob client. |
|
cache/disk
Package disk provides disk-backed implementations of client cache interfaces.
|
Package disk provides disk-backed implementations of client cache interfaces. |
|
mocks
Package mocks provides mock implementations for testing.
|
Package mocks provides mock implementations for testing. |
|
oras
Package oras provides a generic OCI client layer wrapping the ORAS library.
|
Package oras provides a generic OCI client layer wrapping the ORAS library. |