indexer

package
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 16, 2025 License: MIT Imports: 24 Imported by: 0

Documentation

Overview

Package indexer defines indexer server and client. The indexer server is a service that maintain a trusted storage node list, client can fetch the list from server and select nodes in list to upload data.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func InitDefaultIPLocationManager 

func InitDefaultIPLocationManager(config IPLocationConfig)

InitDefaultIPLocationManager initializes the default `IPLocationManager`.

Types

type Client 

type Client struct {
	*rpc.Client
	// contains filtered or unexported fields
}

Client indexer client

func NewClient 

func NewClient(url string, option ...IndexerClientOption) (*Client, error)

NewClient create new indexer client, url is indexer service url

func (*Client) BatchUpload 

func (c *Client) BatchUpload(ctx context.Context, w3Client *web3go.Client, datas []core.IterableData, option ...transfer.BatchUploadOption) (eth_common.Hash, []eth_common.Hash, error)

BatchUpload submit multiple data to 0g storage contract batchly in single on-chain transaction, then transfer the data to the storage nodes selected from indexer service.

func (*Client) Download 

func (c *Client) Download(ctx context.Context, root, filename string, withProof bool) error

Download download file by given data root

func (*Client) DownloadFragments 

func (c *Client) DownloadFragments(ctx context.Context, roots []string, filename string, withProof bool) error

func (*Client) GetFileLocations 

func (c *Client) GetFileLocations(ctx context.Context, root string) ([]*shard.ShardedNode, error)

GetFileLocations return locations info of given file.

func (*Client) GetNodeLocations 

func (c *Client) GetNodeLocations(ctx context.Context) (map[string]*IPLocation, error)

GetNodeLocations return storage nodes with IP location information.

func (*Client) GetShardedNodes 

func (c *Client) GetShardedNodes(ctx context.Context) (ShardedNodes, error)

GetShardedNodes get node list from indexer service

func (*Client) NewDownloaderFromIndexerNodes 

func (c *Client) NewDownloaderFromIndexerNodes(ctx context.Context, root string) (*transfer.Downloader, error)

func (*Client) NewFileSegmentUploaderFromIndexerNodes 

func (c *Client) NewFileSegmentUploaderFromIndexerNodes(
	ctx context.Context, segNum uint64, expectedReplica uint, dropped []string, method string, fullTrusted bool) ([]*transfer.FileSegmentUploader, error)

NewUploaderFromIndexerNodes return a file segment uploader with selected storage nodes from indexer service.

func (*Client) NewUploaderFromIndexerNodes 

func (c *Client) NewUploaderFromIndexerNodes(ctx context.Context, segNum uint64, w3Client *web3go.Client, expectedReplica uint, dropped []string, method string, fullTrusted bool) (*transfer.Uploader, error)

NewUploaderFromIndexerNodes return an uploader with selected storage nodes from indexer service.

func (*Client) SelectNodes 

func (c *Client) SelectNodes(ctx context.Context, expectedReplica uint, dropped []string, method string, fullTrusted bool) (*transfer.SelectedNodes, error)

SelectNodes selects nodes from both trusted and discovered, with discovered max 3/5 of expectedReplica. If discovered cannot meet, all from trusted.

func (*Client) Upload 

func (c *Client) Upload(ctx context.Context, w3Client *web3go.Client, data core.IterableData, option ...transfer.UploadOption) (eth_common.Hash, error)

Upload submit data to 0g storage contract, then transfer the data to the storage nodes selected from indexer service.

func (*Client) UploadFileSegments 

func (c *Client) UploadFileSegments(
	ctx context.Context, fileSeg transfer.FileSegmentsWithProof, option ...transfer.UploadOption) error

UploadFileSegments transfer segment data of a file, which should has already been submitted to the 0g storage contract, to the storage nodes selected from indexer service.

type FileLocation 

type FileLocation struct {
	Url         string            `json:"url"`
	ShardConfig shard.ShardConfig `json:"shardConfig"`
}

type FileLocationCache 

type FileLocationCache struct {
	// contains filtered or unexported fields
}

func InitFileLocationCache 

func InitFileLocationCache(config FileLocationCacheConfig) (cache *FileLocationCache, err error)

func (*FileLocationCache) Close 

func (c *FileLocationCache) Close()

func (*FileLocationCache) GetFileLocations 

func (c *FileLocationCache) GetFileLocations(ctx context.Context, txSeq uint64) ([]*shard.ShardedNode, error)

type FileLocationCacheConfig 

type FileLocationCacheConfig struct {
	CacheSize      int
	Expiry         time.Duration
	DiscoveryNode  string
	DiscoveryPorts []int
}

type IPLocation 

type IPLocation struct {
	City     string `json:"city"`
	Region   string `json:"region"`
	Country  string `json:"country"`
	Location string `json:"loc"`
	Timezone string `json:"timezone"`
}

type IPLocationConfig 

type IPLocationConfig struct {
	CacheFile          string
	CacheWriteInterval time.Duration
	AccessToken        string
}

type IPLocationManager 

type IPLocationManager struct {
	// contains filtered or unexported fields
}

IPLocationManager manages IP locations.

func (*IPLocationManager) All 

func (manager *IPLocationManager) All() map[string]*IPLocation

All returns all cached IP locations.

func (*IPLocationManager) Get 

func (manager *IPLocationManager) Get(ip string) (*IPLocation, bool)

func (*IPLocationManager) Query 

func (manager *IPLocationManager) Query(ip string) (*IPLocation, error)

Query returns the cached IP location if any. Otherwise, retrieve from web API.

type IndexerApi 

type IndexerApi struct {
	Namespace string
}

IndexerApi indexer service configuration

func NewIndexerApi 

func NewIndexerApi() *IndexerApi

NewIndexerApi creates indexer service configuration

func (*IndexerApi) GetFileLocations 

func (api *IndexerApi) GetFileLocations(ctx context.Context, root string) (locations []*shard.ShardedNode, err error)

GetFileLocations return locations info of given file.

func (*IndexerApi) GetNodeLocations 

func (api *IndexerApi) GetNodeLocations(ctx context.Context) (map[string]*IPLocation, error)

GetNodeLocations return IP locations of all nodes.

func (*IndexerApi) GetShardedNodes 

func (api *IndexerApi) GetShardedNodes(ctx context.Context) (ShardedNodes, error)

GetShardedNodes return storage node list

type IndexerClientOption 

type IndexerClientOption struct {
	ProviderOption providers.Option
	LogOption      common.LogOption // log option when uploading data
	FullTrusted    bool             // whether to use full trusted nodes
}

IndexerClientOption indexer client option

type Interface 

type Interface interface {
	GetShardedNodes(ctx context.Context) (ShardedNodes, error)

	GetNodeLocations(ctx context.Context) (map[string]*IPLocation, error)

	GetFileLocations(ctx context.Context, root string) ([]*shard.ShardedNode, error)
}

type NodeManager 

type NodeManager struct {
	// contains filtered or unexported fields
}

NodeManager manages trusted storage nodes and auto discover peers from network.

func InitDefaultNodeManager 

func InitDefaultNodeManager(config NodeManagerConfig) (mgr *NodeManager, err error)

InitDefaultNodeManager initializes the default `NodeManager`.

func (*NodeManager) AddTrustedNodes 

func (nm *NodeManager) AddTrustedNodes(nodes ...string) error

AddTrustedNodes add trusted storage nodes.

func (*NodeManager) Close 

func (nm *NodeManager) Close()

func (*NodeManager) Discovered 

func (nm *NodeManager) Discovered() []*shard.ShardedNode

Discovered returns discovered sharded nodes.

func (*NodeManager) Trusted 

func (nm *NodeManager) Trusted() ([]*shard.ShardedNode, error)

Trusted returns trusted sharded nodes.

func (*NodeManager) TrustedClients 

func (nm *NodeManager) TrustedClients() []*node.ZgsClient

TrustedClients returns trusted clients.

type NodeManagerConfig 

type NodeManagerConfig struct {
	TrustedNodes []string

	DiscoveryNode     string
	DiscoveryInterval time.Duration
	DiscoveryPorts    []int

	UpdateInterval time.Duration
}

type ShardedNodes 

type ShardedNodes struct {
	Trusted    []*shard.ShardedNode `json:"trusted"`
	Discovered []*shard.ShardedNode `json:"discovered"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.