Documentation
¶
Index ¶
- func CompressListingWithZstd(listing *NarListing) ([]byte, error)
- func CompressNarinfo(content string) ([]byte, error)
- func ConvertHashToNix32(hash string) (string, error)
- func EncodeNixBase32(input []byte) string
- func GetBuildLogPath(drvPath string) (string, error)
- func GetPathInfoRecursive(ctx context.Context, storePaths []string, nixEnv []string) (map[string]*PathInfo, error)
- func GetStoreDir(ctx context.Context, nixEnv []string) (string, error)
- func GetStorePathHash(storePath string) (string, error)
- func QueryRealisations(ctx context.Context, pathInfos map[string]*PathInfo, nixEnv []string) (map[string]*RealisationInfo, error)
- type Client
- func (c *Client) CompleteMultipartUpload(ctx context.Context, objectKey, uploadID string, parts []CompletedPart) error
- func (c *Client) CompletePendingClosure(ctx context.Context, closureID string) error
- func (c *Client) CompressAndUploadNAR(ctx context.Context, storePath string, narSize uint64, ...) (*CompressedFileInfo, error)
- func (c *Client) CreatePendingClosure(ctx context.Context, closure string, objects []ObjectWithRefs, verifyS3 bool) (*CreatePendingClosureResponse, error)
- func (c *Client) CreatePendingClosures(ctx context.Context, closures []ClosureInfo) (map[string]PendingObject, map[string]string, error)
- func (c *Client) DoWithRetry(ctx context.Context, req *http.Request) (*http.Response, error)
- func (c *Client) PushPaths(ctx context.Context, paths []string) error
- func (c *Client) RequestMoreParts(ctx context.Context, objectKey, uploadID string, startPartNumber, numParts int) ([]string, error)
- func (c *Client) RunGarbageCollection(ctx context.Context, olderThan string, failedUploadsOlderThan string, ...) (*api.GCStats, error)
- func (c *Client) SetDebugHTTP(enabled bool)
- func (c *Client) SignAndUploadNarinfos(ctx context.Context, narinfosByClosureID map[string]map[string]NarinfoMetadata, ...) error
- func (c *Client) SignPendingClosure(ctx context.Context, closureID string, narinfos map[string]NarinfoMetadata) (map[string][]string, error)
- func (c *Client) UploadBuildLogToPresignedURL(ctx context.Context, presignedURL string, ...) error
- func (c *Client) UploadBytesToPresignedURL(ctx context.Context, presignedURL string, data []byte) error
- func (c *Client) UploadBytesToPresignedURLWithHeaders(ctx context.Context, presignedURL string, data []byte, ...) error
- func (c *Client) UploadListingToPresignedURL(ctx context.Context, presignedURL string, listing *NarListing) error
- func (c *Client) UploadPendingObjects(ctx context.Context, uploadCtx *UploadContext) (map[string]NarinfoMetadata, error)
- type ClosureInfo
- type CompletedPart
- type CompressedBuildLogInfo
- type CompressedFileInfo
- type ContentAddress
- type CreatePendingClosureResponse
- type Hash
- type MultipartUploadInfo
- type NarListing
- type NarListingEntry
- type NarinfoMetadata
- type ObjectType
- type ObjectWithRefs
- type PathInfo
- type PendingObject
- type PrepareClosuresResult
- type RealisationInfo
- type RetryConfig
- type UploadContext
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CompressListingWithZstd ¶
func CompressListingWithZstd(listing *NarListing) ([]byte, error)
CompressListingWithZstd compresses a NAR listing as JSON with zstd compression. This matches Nix's compression approach and provides better performance than brotli. It reuses the existing zstdEncoderPool from nar_upload.go.
func CompressNarinfo ¶
CompressNarinfo compresses narinfo content using zstd with encoder pooling.
func ConvertHashToNix32 ¶
ConvertHashToNix32 converts a hash from SRI format (sha256-base64) or Nix32 format (sha256:nix32) to Nix32 format (sha256:nix32). If the hash is already in Nix32 format, it returns it unchanged.
func EncodeNixBase32 ¶
EncodeNixBase32 encodes bytes into Nix's base32 format. This implementation is based on Nix's BaseNix32::encode in src/libutil/base-nix-32.cc.
func GetBuildLogPath ¶
GetBuildLogPath finds the build log file for a derivation path. It checks for both plain and .bz2 compressed logs. Returns the path to the log file if found, or an empty string if not found.
func GetPathInfoRecursive ¶
func GetPathInfoRecursive(ctx context.Context, storePaths []string, nixEnv []string) (map[string]*PathInfo, error)
GetPathInfoRecursive queries Nix for path info including all dependencies.
func GetStoreDir ¶
GetStoreDir determines the Nix store directory path. It checks in order: 1. NIX_STORE_DIR environment variable (from nixEnv if provided) 2. Queries nix command 3. Falls back to default "/nix/store" Returns the store directory (e.g., "/nix/store").
func GetStorePathHash ¶
GetStorePathHash extracts the hash from a store path. e.g., "/nix/store/abc123-name" -> "abc123".
func QueryRealisations ¶
func QueryRealisations(ctx context.Context, pathInfos map[string]*PathInfo, nixEnv []string) (map[string]*RealisationInfo, error)
QueryRealisations queries realisations from Nix's local database using `nix realisation info`. It only queries paths that have the CA field set, as non-CA paths don't have realisations. Returns a map from realisation key ("realisations/<id>.doi") to RealisationInfo.
Types ¶
type Client ¶
type Client struct {
MaxConcurrentNARUploads int // Maximum number of concurrent uploads (0 = unlimited)
NixEnv []string // Optional environment variables for nix commands (for testing)
Retry RetryConfig // Retry configuration for HTTP requests
VerifyS3Integrity bool // Enable S3 integrity checking when creating pending closures
DebugHTTP bool // Enable HTTP request/response debug logging
// contains filtered or unexported fields
}
Client handles uploads to the niks3 server.
func NewClient ¶
NewClient creates a new upload client. The default MaxConcurrentNARUploads is set to 16, optimized for I/O-bound upload workloads. This is comparable to browser HTTP/2 connection limits and Cachix's default of 8.
TODO: Test this value in various network setups (local network, high-latency WAN, rate-limited connections) to determine optimal defaults for different scenarios.
func (*Client) CompleteMultipartUpload ¶
func (c *Client) CompleteMultipartUpload(ctx context.Context, objectKey, uploadID string, parts []CompletedPart) error
CompleteMultipartUpload completes a multipart upload.
func (*Client) CompletePendingClosure ¶
CompletePendingClosure marks a closure as complete after all objects have been uploaded. This should be called after narinfos have been signed and uploaded.
func (*Client) CompressAndUploadNAR ¶
func (c *Client) CompressAndUploadNAR(ctx context.Context, storePath string, narSize uint64, pendingObj PendingObject, objectKey string) (*CompressedFileInfo, error)
CompressAndUploadNAR compresses a NAR and uploads it using multipart upload. It also generates a directory listing during serialization.
func (*Client) CreatePendingClosure ¶
func (c *Client) CreatePendingClosure(ctx context.Context, closure string, objects []ObjectWithRefs, verifyS3 bool) (*CreatePendingClosureResponse, error)
CreatePendingClosure creates a pending closure and returns upload URLs.
func (*Client) CreatePendingClosures ¶
func (c *Client) CreatePendingClosures(ctx context.Context, closures []ClosureInfo) (map[string]PendingObject, map[string]string, error)
CreatePendingClosures creates pending closures and returns all pending objects and closure ID to narinfo key mapping.
func (*Client) DoWithRetry ¶
DoWithRetry executes an HTTP request with exponential backoff retry logic. The request body will be read and stored for retries if necessary.
func (*Client) RequestMoreParts ¶
func (c *Client) RequestMoreParts(ctx context.Context, objectKey, uploadID string, startPartNumber, numParts int) ([]string, error)
RequestMoreParts requests additional part URLs for an existing multipart upload.
func (*Client) RunGarbageCollection ¶
func (c *Client) RunGarbageCollection(ctx context.Context, olderThan string, failedUploadsOlderThan string, force bool) (*api.GCStats, error)
RunGarbageCollection triggers garbage collection on the server for closures older than the specified duration. If force is true, objects will be deleted immediately without a grace period (may be dangerous). failedUploadsOlderThan specifies how old failed uploads must be before cleanup (server defaults to "6h" if empty). Returns statistics about the garbage collection run.
func (*Client) SetDebugHTTP ¶
SetDebugHTTP enables or disables HTTP request/response logging. When enabled, wraps the HTTP client transport with a logging transport.
func (*Client) SignAndUploadNarinfos ¶
func (c *Client) SignAndUploadNarinfos(ctx context.Context, narinfosByClosureID map[string]map[string]NarinfoMetadata, pendingObjects map[string]PendingObject) error
SignAndUploadNarinfos signs narinfos on the server and uploads them to S3 in parallel.
func (*Client) SignPendingClosure ¶
func (c *Client) SignPendingClosure(ctx context.Context, closureID string, narinfos map[string]NarinfoMetadata) (map[string][]string, error)
SignPendingClosure sends narinfo metadata to the server for signing and returns signatures.
func (*Client) UploadBuildLogToPresignedURL ¶
func (c *Client) UploadBuildLogToPresignedURL(ctx context.Context, presignedURL string, compressedInfo *CompressedBuildLogInfo) error
UploadBuildLogToPresignedURL uploads a compressed build log with Content-Encoding header. This follows Nix's convention for compressed build logs stored at log/<drvPath>. The compressedInfo must point to a temporary file created by CompressBuildLog.
func (*Client) UploadBytesToPresignedURL ¶
func (c *Client) UploadBytesToPresignedURL(ctx context.Context, presignedURL string, data []byte) error
UploadBytesToPresignedURL uploads bytes to a presigned URL.
func (*Client) UploadBytesToPresignedURLWithHeaders ¶
func (c *Client) UploadBytesToPresignedURLWithHeaders(ctx context.Context, presignedURL string, data []byte, headers map[string]string) error
UploadBytesToPresignedURLWithHeaders uploads bytes to a presigned URL with optional custom headers.
func (*Client) UploadListingToPresignedURL ¶
func (c *Client) UploadListingToPresignedURL(ctx context.Context, presignedURL string, listing *NarListing) error
UploadListingToPresignedURL compresses a NAR listing with zstd and uploads it with Content-Encoding header. The listing is stored as a .ls file, compatible with Nix's lazy NAR accessor format.
func (*Client) UploadPendingObjects ¶
func (c *Client) UploadPendingObjects(ctx context.Context, uploadCtx *UploadContext) (map[string]NarinfoMetadata, error)
UploadPendingObjects uploads all pending objects (NARs, .ls files, build logs, and realisations). Returns narinfo metadata for each closure to be signed and uploaded by the server. Uses a unified worker pool where: - Logs and realisations upload immediately (independent) - NARs upload and queue their listings when complete - Narinfo metadata is collected (not uploaded) for server-side signing.
type ClosureInfo ¶
type ClosureInfo struct {
NarinfoKey string
Objects []ObjectWithRefs
}
ClosureInfo represents a closure with its associated objects.
type CompletedPart ¶
CompletedPart represents a completed multipart part.
type CompressedBuildLogInfo ¶
type CompressedBuildLogInfo struct {
TempFile string // Path to temporary file containing compressed log
Size int64 // Size of compressed log
}
CompressedBuildLogInfo contains information about the compressed build log.
func CompressBuildLog ¶
func CompressBuildLog(logPath string) (*CompressedBuildLogInfo, error)
CompressBuildLog reads and compresses a build log file to a temporary file. It automatically decompresses .bz2 source files and recompresses with zstd. Returns info about the compressed temp file. The caller must call Cleanup() when done.
func (*CompressedBuildLogInfo) Cleanup ¶
func (info *CompressedBuildLogInfo) Cleanup() error
Cleanup removes the temporary compressed log file. It's safe to call multiple times.
type CompressedFileInfo ¶
type CompressedFileInfo struct {
Listing *NarListing // Directory listing (if generated)
}
CompressedFileInfo contains information about a compressed file.
type ContentAddress ¶
type ContentAddress struct {
// contains filtered or unexported fields
}
ContentAddress represents a Nix content address. It supports both the old string format (e.g., "fixed:r:sha256:abc...") and the new structured format from Nix 2.33+.
func (*ContentAddress) String ¶
func (ca *ContentAddress) String() string
String returns the content address in the narinfo-compatible string format. Nix narinfo files use these prefixes:
- "text:sha256:..." for text
- "fixed:sha256:..." for flat
- "fixed:r:sha256:..." for nar (recursive)
- "fixed:git:sha256:..." for git
The hash is in nix32 format (sha256:base32hash).
func (*ContentAddress) UnmarshalJSON ¶
func (ca *ContentAddress) UnmarshalJSON(data []byte) error
UnmarshalJSON implements custom JSON unmarshaling to support both old string format and new structured format from nix path-info.
type CreatePendingClosureResponse ¶
type CreatePendingClosureResponse struct {
ID string `json:"id"`
StartedAt string `json:"started_at"`
PendingObjects map[string]PendingObject `json:"pending_objects"`
}
CreatePendingClosureResponse is the response from creating a pending closure.
type Hash ¶
type Hash struct {
// contains filtered or unexported fields
}
Hash represents a Nix hash value. It supports both the old string format (e.g., "sha256-base64hash") and the new structured format from Nix 2.33+.
func (*Hash) String ¶
String returns the hash in SRI format (e.g., "sha256-base64hash"). This format is compatible with the existing ConvertHashToNix32 function.
func (*Hash) UnmarshalJSON ¶
UnmarshalJSON implements custom JSON unmarshaling to support both old string format and new structured format from nix path-info.
type MultipartUploadInfo ¶
type MultipartUploadInfo struct {
UploadID string `json:"upload_id"`
PartURLs []string `json:"part_urls"`
}
MultipartUploadInfo contains multipart upload information.
type NarListing ¶
type NarListing struct {
Version int `json:"version"`
Root NarListingEntry `json:"root"`
}
NarListing represents the directory structure of a NAR archive in Nix's format.
func DumpPathWithListing ¶
func DumpPathWithListing(w io.Writer, path string) (*NarListing, error)
DumpPathWithListing serializes a path to NAR format and returns the directory listing. The listing is compatible with Nix's .ls format.
func GenerateListingOnly ¶
func GenerateListingOnly(path string) (*NarListing, error)
GenerateListingOnly creates a directory listing by walking the filesystem without serializing the NAR. This is much faster for deduplicated NARs where we only need the listing structure.
type NarListingEntry ¶
type NarListingEntry struct {
Type string `json:"type"` // "regular", "directory", "symlink"
Size *uint64 `json:"size,omitempty"`
Executable *bool `json:"executable,omitempty"`
NarOffset *uint64 `json:"narOffset,omitempty"` //nolint:tagliatelle // matches Nix's JSON format
Entries map[string]NarListingEntry `json:"entries,omitempty"`
Target *string `json:"target,omitempty"`
}
NarListingEntry represents a file, directory, or symlink in a NAR listing.
type NarinfoMetadata ¶
type NarinfoMetadata struct {
StorePath string `json:"store_path"`
URL string `json:"url"` // e.g., "nar/xxxxx.nar.zst"
Compression string `json:"compression"` // e.g., "zstd"
NarHash string `json:"nar_hash"` // e.g., "sha256:xxxxx"
NarSize uint64 `json:"nar_size"` // Uncompressed NAR size
References []string `json:"references"` // Store paths (with /nix/store prefix)
Deriver *string `json:"deriver,omitempty"`
Signatures []string `json:"signatures,omitempty"`
CA *string `json:"ca,omitempty"`
}
NarinfoMetadata contains metadata for a narinfo file to be signed by the server.
type ObjectType ¶
type ObjectType string
ObjectType classifies cache objects by their purpose and upload strategy.
const ( ObjectTypeNarinfo ObjectType = "narinfo" ObjectTypeListing ObjectType = "listing" ObjectTypeBuildLog ObjectType = "build_log" ObjectTypeNAR ObjectType = "nar" ObjectTypeRealisation ObjectType = "realisation" )
type ObjectWithRefs ¶
type ObjectWithRefs struct {
Key string `json:"key"`
Type ObjectType `json:"type"`
Refs []string `json:"refs"`
NarSize *uint64 `json:"nar_size,omitempty"` // For estimating multipart parts
}
ObjectWithRefs represents an object with its dependencies.
type PathInfo ¶
type PathInfo struct {
Path string `json:"-"`
//nolint:tagliatelle // narHash and narSize are defined by Nix's JSON format
NarHash Hash `json:"narHash"`
//nolint:tagliatelle // narHash and narSize are defined by Nix's JSON format
NarSize uint64 `json:"narSize"`
References []string `json:"references"`
Deriver *string `json:"deriver,omitempty"`
Signatures []string `json:"signatures,omitempty"`
CA *ContentAddress `json:"ca,omitempty"`
}
PathInfo represents Nix path information.
type PendingObject ¶
type PendingObject struct {
Type string `json:"type"` // Object type (narinfo, listing, build_log, nar)
PresignedURL string `json:"presigned_url,omitempty"` // For small files
MultipartInfo *MultipartUploadInfo `json:"multipart_info,omitempty"` // For large files
}
PendingObject contains upload information for an object.
type PrepareClosuresResult ¶
type PrepareClosuresResult struct {
Closures []ClosureInfo
PathInfoByHash map[string]*PathInfo
NARKeyToHash map[string]string // Maps NAR object key -> store path hash
LogPathsByKey map[string]string // Maps log object key -> local log file path
RealisationsByKey map[string]*RealisationInfo // Maps realisation key -> realisation info
}
PrepareClosuresResult contains the result of preparing closures.
func PrepareClosures ¶
func PrepareClosures(ctx context.Context, topLevelPaths []string, pathInfos map[string]*PathInfo, nixEnv []string) (*PrepareClosuresResult, error)
PrepareClosures prepares closures from path info, including NAR, .ls, narinfo, build log, and realisation objects. Build logs are automatically discovered for output paths and included by default. Realisations are queried for CA derivations and included automatically. topLevelPaths specifies which paths are closure roots - one ClosureInfo is created per top-level path.
type RealisationInfo ¶
type RealisationInfo struct {
ID string `json:"id"` // "sha256:hash!outputName"
OutPath string `json:"outPath"` //nolint:tagliatelle // outPath is defined by Nix's JSON format
Signatures []string `json:"signatures,omitempty"`
DependentRealisations map[string]string `json:"dependentRealisations,omitempty"` //nolint:tagliatelle
}
RealisationInfo represents Nix realisation information for CA derivations.
type RetryConfig ¶
type RetryConfig struct {
MaxRetries int // Maximum number of retry attempts (0 = no retries)
InitialBackoff time.Duration // Initial backoff duration
MaxBackoff time.Duration // Maximum backoff duration
Multiplier float64 // Backoff multiplier for each retry
Jitter float64 // Random jitter factor (0.0-1.0)
}
RetryConfig holds retry configuration for HTTP requests.
func DefaultRetryConfig ¶
func DefaultRetryConfig() RetryConfig
DefaultRetryConfig returns sensible defaults for retry configuration.
type UploadContext ¶
type UploadContext struct {
PendingObjects map[string]PendingObject
PathInfoByHash map[string]*PathInfo
NARKeyToHash map[string]string
LogPathsByKey map[string]string
RealisationsByKey map[string]*RealisationInfo
}
UploadContext contains all the context needed for uploading objects.