Skip to content

GitLab

Commit d68ecf1b authored by laser's avatar laser
Browse files

update bindings, submodule, and add lifecycle test

parent ff9bb895
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ d68ecf1b
**/*.h **/*.h
**/*.a **/*.a
**/*.pc **/*.pc
.install-rust-fil-sector-builder
Makefile 0 → 100644
View file @ d68ecf1b
DEPS:=sector_builder_ffi.h sector_builder_ffi.pc sector_builder_ffi.a
all: $(DEPS)
.PHONY: all
$(DEPS): .install-rust-fil-sector-builder ;
.install-rust-fil-sector-builder: rust-fil-sector-builder
./install-rust-fil-sector-builder
@touch $@
clean:
rm -rf $(DEPS) .install-rust-fil-sector-builder
.PHONY: clean
binding/cgo_bindings.go → bindings.go
View file @ d68ecf1b
// +build !windows package go_sectorbuilder
package sectorbuilder
import ( import (
"time" "time"
...@@ -10,9 +8,9 @@ import ( ...@@ -10,9 +8,9 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi // #cgo LDFLAGS: ${SRCDIR}/libsector_builder_ffi.a
// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc // #cgo pkg-config: ${SRCDIR}/sector_builder_ffi.pc
// #include "../include/sector_builder_ffi.h" // #include "./sector_builder_ffi.h"
import "C" import "C"
var log = logging.Logger("libsectorbuilder") // nolint: deadcode var log = logging.Logger("libsectorbuilder") // nolint: deadcode
...@@ -24,6 +22,9 @@ func elapsed(what string) func() { ...@@ -24,6 +22,9 @@ func elapsed(what string) func() {
} }
} }
// CommitmentBytesLen is the number of bytes in a CommR, CommD, CommP, and CommRStar.
const CommitmentBytesLen = 32
// StagedSectorMetadata is a sector into which we write user piece-data before // StagedSectorMetadata is a sector into which we write user piece-data before
// sealing. Note: SectorID is unique across all staged and sealed sectors for a // sealing. Note: SectorID is unique across all staged and sealed sectors for a
// storage miner actor. // storage miner actor.
...@@ -35,13 +36,13 @@ type StagedSectorMetadata struct { ...@@ -35,13 +36,13 @@ type StagedSectorMetadata struct {
// sector has progressed. // sector has progressed.
type SectorSealingStatus struct { type SectorSealingStatus struct {
SectorID uint64 SectorID uint64
SealStatusCode uint8 // Sealed = 0, Pending = 1, Failed = 2, Sealing = 3 SealStatusCode uint8 // Sealed = 0, Pending = 1, Failed = 2, Sealing = 3
SealErrorMsg string // will be nil unless SealStatusCode == 2 SealErrorMsg string // will be nil unless SealStatusCode == 2
CommD [32]byte // will be empty unless SealStatusCode == 0 CommD [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
CommR [32]byte // will be empty unless SealStatusCode == 0 CommR [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
CommRStar [32]byte // will be empty unless SealStatusCode == 0 CommRStar [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
Proof []byte // will be empty unless SealStatusCode == 0 Proof []byte // will be empty unless SealStatusCode == 0
Pieces []PieceMetadata // will be empty unless SealStatusCode == 0 Pieces []PieceMetadata // will be empty unless SealStatusCode == 0
} }
// PieceMetadata represents a piece stored by the sector builder. // PieceMetadata represents a piece stored by the sector builder.
...@@ -49,15 +50,16 @@ type PieceMetadata struct { ...@@ -49,15 +50,16 @@ type PieceMetadata struct {
Key string Key string
Size uint64 Size uint64
InclusionProof []byte InclusionProof []byte
CommP [CommitmentBytesLen]byte
} }
// VerifySeal returns true if the sealing operation from which its inputs were // VerifySeal returns true if the sealing operation from which its inputs were
// derived was valid, and false if not. // derived was valid, and false if not.
func VerifySeal( func VerifySeal(
sectorSize uint64, sectorSize uint64,
commR [32]byte, commR [CommitmentBytesLen]byte,
commD [32]byte, commD [CommitmentBytesLen]byte,
commRStar [32]byte, commRStar [CommitmentBytesLen]byte,
proverID [31]byte, proverID [31]byte,
sectorID [31]byte, sectorID [31]byte,
proof []byte, proof []byte,
...@@ -83,16 +85,16 @@ func VerifySeal( ...@@ -83,16 +85,16 @@ func VerifySeal(
defer C.free(sectorIDCbytes) defer C.free(sectorIDCbytes)
// a mutable pointer to a VerifySealResponse C-struct // a mutable pointer to a VerifySealResponse C-struct
resPtr := (*C.sector_builder_ffi_VerifySealResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_seal( resPtr := C.sector_builder_ffi_verify_seal(
C.uint64_t(sectorSize), C.uint64_t(sectorSize),
(*[32]C.uint8_t)(commRCBytes), (*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
(*[32]C.uint8_t)(commDCBytes), (*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
(*[32]C.uint8_t)(commRStarCBytes), (*[CommitmentBytesLen]C.uint8_t)(commRStarCBytes),
(*[31]C.uint8_t)(proverIDCBytes), (*[31]C.uint8_t)(proverIDCBytes),
(*[31]C.uint8_t)(sectorIDCbytes), (*[31]C.uint8_t)(sectorIDCbytes),
(*C.uint8_t)(proofCBytes), (*C.uint8_t)(proofCBytes),
C.size_t(len(proof)), C.size_t(len(proof)),
))) )
defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr) defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -106,8 +108,8 @@ func VerifySeal( ...@@ -106,8 +108,8 @@ func VerifySeal(
// inputs were derived was valid, and false if not. // inputs were derived was valid, and false if not.
func VerifyPoSt( func VerifyPoSt(
sectorSize uint64, sectorSize uint64,
sortedCommRs [][32]byte, sortedCommRs [][CommitmentBytesLen]byte,
challengeSeed [32]byte, challengeSeed [CommitmentBytesLen]byte,
proofs [][]byte, proofs [][]byte,
faults []uint64, faults []uint64,
) (bool, error) { ) (bool, error) {
...@@ -123,9 +125,9 @@ func VerifyPoSt( ...@@ -123,9 +125,9 @@ func VerifyPoSt(
commRs := sortedCommRs commRs := sortedCommRs
// flattening the byte slice makes it easier to copy into the C heap // flattening the byte slice makes it easier to copy into the C heap
flattened := make([]byte, 32*len(commRs)) flattened := make([]byte, CommitmentBytesLen*len(commRs))
for idx, commR := range commRs { for idx, commR := range commRs {
copy(flattened[(32*idx):(32*(1+idx))], commR[:]) copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
} }
// copy bytes from Go to C heap // copy bytes from Go to C heap
...@@ -143,17 +145,17 @@ func VerifyPoSt( ...@@ -143,17 +145,17 @@ func VerifyPoSt(
defer C.free(unsafe.Pointer(faultsPtr)) defer C.free(unsafe.Pointer(faultsPtr))
// a mutable pointer to a VerifyPoStResponse C-struct // a mutable pointer to a VerifyPoStResponse C-struct
resPtr := (*C.sector_builder_ffi_VerifyPoStResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_post( resPtr := C.sector_builder_ffi_verify_post(
C.uint64_t(sectorSize), C.uint64_t(sectorSize),
proofPartitions, proofPartitions,
(*C.uint8_t)(flattenedCommRsCBytes), (*C.uint8_t)(flattenedCommRsCBytes),
C.size_t(len(flattened)), C.size_t(len(flattened)),
(*[32]C.uint8_t)(challengeSeedCBytes), (*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
proofsPtr, proofsPtr,
proofsLen, proofsLen,
faultsPtr, faultsPtr,
faultsSize, faultsSize,
))) )
defer C.sector_builder_ffi_destroy_verify_post_response(resPtr) defer C.sector_builder_ffi_destroy_verify_post_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -203,7 +205,7 @@ func InitSectorBuilder( ...@@ -203,7 +205,7 @@ func InitSectorBuilder(
return nil, errors.Wrap(err, "failed to get sector class") return nil, errors.Wrap(err, "failed to get sector class")
} }
resPtr := (*C.sector_builder_ffi_InitSectorBuilderResponse)(unsafe.Pointer(C.sector_builder_ffi_init_sector_builder( resPtr := C.sector_builder_ffi_init_sector_builder(
class, class,
C.uint64_t(lastUsedSectorID), C.uint64_t(lastUsedSectorID),
cMetadataDir, cMetadataDir,
...@@ -211,7 +213,7 @@ func InitSectorBuilder( ...@@ -211,7 +213,7 @@ func InitSectorBuilder(
cSealedSectorDir, cSealedSectorDir,
cStagedSectorDir, cStagedSectorDir,
C.uint8_t(maxNumOpenStagedSectors), C.uint8_t(maxNumOpenStagedSectors),
))) )
defer C.sector_builder_ffi_destroy_init_sector_builder_response(resPtr) defer C.sector_builder_ffi_destroy_init_sector_builder_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -246,12 +248,12 @@ func AddPiece( ...@@ -246,12 +248,12 @@ func AddPiece(
cPiecePath := C.CString(piecePath) cPiecePath := C.CString(piecePath)
defer C.free(unsafe.Pointer(cPiecePath)) defer C.free(unsafe.Pointer(cPiecePath))
resPtr := (*C.sector_builder_ffi_AddPieceResponse)(unsafe.Pointer(C.sector_builder_ffi_add_piece( resPtr := C.sector_builder_ffi_add_piece(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), (*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
cPieceKey, cPieceKey,
C.uint64_t(pieceSize), C.uint64_t(pieceSize),
cPiecePath, cPiecePath,
))) )
defer C.sector_builder_ffi_destroy_add_piece_response(resPtr) defer C.sector_builder_ffi_destroy_add_piece_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -270,7 +272,10 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string) ...@@ -270,7 +272,10 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string)
cPieceKey := C.CString(pieceKey) cPieceKey := C.CString(pieceKey)
defer C.free(unsafe.Pointer(cPieceKey)) defer C.free(unsafe.Pointer(cPieceKey))
resPtr := (*C.sector_builder_ffi_ReadPieceFromSealedSectorResponse)(unsafe.Pointer(C.sector_builder_ffi_read_piece_from_sealed_sector((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), cPieceKey))) resPtr := C.sector_builder_ffi_read_piece_from_sealed_sector(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
cPieceKey,
)
defer C.sector_builder_ffi_destroy_read_piece_from_sealed_sector_response(resPtr) defer C.sector_builder_ffi_destroy_read_piece_from_sealed_sector_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -284,7 +289,7 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string) ...@@ -284,7 +289,7 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string)
func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error { func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error {
defer elapsed("SealAllStagedSectors")() defer elapsed("SealAllStagedSectors")()
resPtr := (*C.sector_builder_ffi_SealAllStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_seal_all_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr)))) resPtr := C.sector_builder_ffi_seal_all_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))
defer C.sector_builder_ffi_destroy_seal_all_staged_sectors_response(resPtr) defer C.sector_builder_ffi_destroy_seal_all_staged_sectors_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -298,14 +303,14 @@ func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error { ...@@ -298,14 +303,14 @@ func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error {
func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadata, error) { func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadata, error) {
defer elapsed("GetAllStagedSectors")() defer elapsed("GetAllStagedSectors")()
resPtr := (*C.sector_builder_ffi_GetStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_get_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr)))) resPtr := C.sector_builder_ffi_get_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))
defer C.sector_builder_ffi_destroy_get_staged_sectors_response(resPtr) defer C.sector_builder_ffi_destroy_get_staged_sectors_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
return nil, errors.New(C.GoString(resPtr.error_msg)) return nil, errors.New(C.GoString(resPtr.error_msg))
} }
meta, err := goStagedSectorMetadata((*C.sector_builder_ffi_FFIStagedSectorMetadata)(unsafe.Pointer(resPtr.sectors_ptr)), resPtr.sectors_len) meta, err := goStagedSectorMetadata(resPtr.sectors_ptr, resPtr.sectors_len)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -319,7 +324,10 @@ func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadat ...@@ -319,7 +324,10 @@ func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadat
func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64) (SectorSealingStatus, error) { func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64) (SectorSealingStatus, error) {
defer elapsed("GetSectorSealingStatusByID")() defer elapsed("GetSectorSealingStatusByID")()
resPtr := (*C.sector_builder_ffi_GetSealStatusResponse)(unsafe.Pointer(C.sector_builder_ffi_get_seal_status((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), C.uint64_t(sectorID)))) resPtr := C.sector_builder_ffi_get_seal_status(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
C.uint64_t(sectorID),
)
defer C.sector_builder_ffi_destroy_get_seal_status_response(resPtr) defer C.sector_builder_ffi_destroy_get_seal_status_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -333,16 +341,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64 ...@@ -333,16 +341,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
} else if resPtr.seal_status_code == C.Sealing { } else if resPtr.seal_status_code == C.Sealing {
return SectorSealingStatus{SealStatusCode: 3}, nil return SectorSealingStatus{SealStatusCode: 3}, nil
} else if resPtr.seal_status_code == C.Sealed { } else if resPtr.seal_status_code == C.Sealed {
commRSlice := goBytes(&resPtr.comm_r[0], 32) commRSlice := goBytes(&resPtr.comm_r[0], CommitmentBytesLen)
var commR [32]byte var commR [CommitmentBytesLen]byte
copy(commR[:], commRSlice) copy(commR[:], commRSlice)
commDSlice := goBytes(&resPtr.comm_d[0], 32) commDSlice := goBytes(&resPtr.comm_d[0], CommitmentBytesLen)
var commD [32]byte var commD [CommitmentBytesLen]byte
copy(commD[:], commDSlice) copy(commD[:], commDSlice)
commRStarSlice := goBytes(&resPtr.comm_r_star[0], 32) commRStarSlice := goBytes(&resPtr.comm_r_star[0], CommitmentBytesLen)
var commRStar [32]byte var commRStar [CommitmentBytesLen]byte
copy(commRStar[:], commRStarSlice) copy(commRStar[:], commRStarSlice)
proof := goBytes(resPtr.proof_ptr, resPtr.proof_len) proof := goBytes(resPtr.proof_ptr, resPtr.proof_len)
...@@ -370,16 +378,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64 ...@@ -370,16 +378,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
// GeneratePoSt produces a proof-of-spacetime for the provided replica commitments. // GeneratePoSt produces a proof-of-spacetime for the provided replica commitments.
func GeneratePoSt( func GeneratePoSt(
sectorBuilderPtr unsafe.Pointer, sectorBuilderPtr unsafe.Pointer,
sortedCommRs [][32]byte, sortedCommRs [][CommitmentBytesLen]byte,
challengeSeed [32]byte, challengeSeed [CommitmentBytesLen]byte,
) ([][]byte, []uint64, error) { ) ([][]byte, []uint64, error) {
defer elapsed("GeneratePoSt")() defer elapsed("GeneratePoSt")()
// flattening the byte slice makes it easier to copy into the C heap // flattening the byte slice makes it easier to copy into the C heap
commRs := sortedCommRs commRs := sortedCommRs
flattened := make([]byte, 32*len(commRs)) flattened := make([]byte, CommitmentBytesLen*len(commRs))
for idx, commR := range commRs { for idx, commR := range commRs {
copy(flattened[(32*idx):(32*(1+idx))], commR[:]) copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
} }
// copy the Go byte slice into C memory // copy the Go byte slice into C memory
...@@ -389,7 +397,12 @@ func GeneratePoSt( ...@@ -389,7 +397,12 @@ func GeneratePoSt(
challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0]) challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0])
// a mutable pointer to a GeneratePoStResponse C-struct // a mutable pointer to a GeneratePoStResponse C-struct
resPtr := (*C.sector_builder_ffi_GeneratePoStResponse)(unsafe.Pointer(C.sector_builder_ffi_generate_post((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), (*C.uint8_t)(cflattened), C.size_t(len(flattened)), (*[32]C.uint8_t)(challengeSeedPtr)))) resPtr := C.sector_builder_ffi_generate_post(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
(*C.uint8_t)(cflattened),
C.size_t(len(flattened)),
(*[CommitmentBytesLen]C.uint8_t)(challengeSeedPtr),
)
defer C.sector_builder_ffi_destroy_generate_post_response(resPtr) defer C.sector_builder_ffi_destroy_generate_post_response(resPtr)
if resPtr.status_code != 0 { if resPtr.status_code != 0 {
...@@ -403,3 +416,52 @@ func GeneratePoSt( ...@@ -403,3 +416,52 @@ func GeneratePoSt(
return proofs, goUint64s(resPtr.faults_ptr, resPtr.faults_len), nil return proofs, goUint64s(resPtr.faults_ptr, resPtr.faults_len), nil
} }
// VerifyPieceInclusionProof returns true if the piece inclusion proof is valid
// with the given arguments.
func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP [CommitmentBytesLen]byte, commD [CommitmentBytesLen]byte, proof []byte) (bool, error) {
commDCBytes := C.CBytes(commD[:])
defer C.free(commDCBytes)
commPCBytes := C.CBytes(commP[:])
defer C.free(commPCBytes)
pieceInclusionProofCBytes := C.CBytes(proof)
defer C.free(pieceInclusionProofCBytes)
resPtr := C.sector_builder_ffi_verify_piece_inclusion_proof(
(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commPCBytes),
(*C.uint8_t)(pieceInclusionProofCBytes),
C.size_t(len(proof)),
C.uint64_t(pieceSize),
C.uint64_t(sectorSize),
)
defer C.sector_builder_ffi_destroy_verify_piece_inclusion_proof_response(resPtr)
if resPtr.status_code != 0 {
return false, errors.New(C.GoString(resPtr.error_msg))
}
return bool(resPtr.is_valid), nil
}
// GeneratePieceCommitment produces a piece commitment for the provided data
// stored at a given piece path.
func GeneratePieceCommitment(piecePath string, pieceSize uint64) (commP [CommitmentBytesLen]byte, err error) {
cPiecePath := C.CString(piecePath)
defer C.free(unsafe.Pointer(cPiecePath))
resPtr := C.sector_builder_ffi_generate_piece_commitment(cPiecePath, C.uint64_t(pieceSize))
defer C.sector_builder_ffi_destroy_generate_piece_commitment_response(resPtr)
if resPtr.status_code != 0 {
return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
}
commPSlice := goBytes(&resPtr.comm_p[0], CommitmentBytesLen)
var commitment [CommitmentBytesLen]byte
copy(commitment[:], commPSlice)
return commitment, nil
}
bindings_test.go 0 → 100644
View file @ d68ecf1b
package go_sectorbuilder_test
import (
"bytes"
"crypto/rand"
"encoding/binary"
"errors"
"io"
"io/ioutil"
"os"
"testing"
"time"
"unsafe"
sb "github.com/filecoin-project/go-sectorbuilder"
"github.com/stretchr/testify/require"
)
func TestSectorBuilderLifecycle(t *testing.T) {
metadataDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(metadataDir))
sealedSectorDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(sealedSectorDir))
stagedSectorDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(stagedSectorDir))
ptr, err := sb.InitSectorBuilder(1024, 2, 1, 0, metadataDir, [31]byte{}, sealedSectorDir, stagedSectorDir, 1)
require.NoError(t, err)
defer sb.DestroySectorBuilder(ptr)
// compute the max user-bytes that can fit into a staged sector such that
// bit-padding ("preprocessing") expands the file to $SECTOR_SIZE
maxPieceSize := sb.GetMaxUserBytesPerStagedSector(1024)
// create a piece which consumes all available space in a new, staged
// sector
pieceBytes := make([]byte, maxPieceSize)
_, err = io.ReadFull(rand.Reader, pieceBytes)
require.NoError(t, err)
piecePath := requireTempFilePath(t, bytes.NewReader(pieceBytes))
// generate piece commitment
commP, err := sb.GeneratePieceCommitment(piecePath, maxPieceSize)
require.NoError(t, err)
// write a piece to a staged sector, reducing remaining space to 0 and
// triggering the seal job
sectorID, err := sb.AddPiece(ptr, "snoqualmie", maxPieceSize, piecePath)
require.NoError(t, err)
// block until Groth parameter cache is (lazily) hydrated and sector has
// been sealed (or timeout)
status, err := pollForSectorSealingStatus(ptr, sectorID, 0, time.Minute*30)
require.NoError(t, err)
require.Equal(t, 1, len(status.Pieces), "expected to see the one piece we added")
// verify the seal proof
isValid, err := sb.VerifySeal(1024, status.CommR, status.CommD, status.CommRStar, [31]byte{}, sectorIdAsBytes(sectorID), status.Proof)
require.NoError(t, err)
require.True(t, isValid)
// verify the piece inclusion proof
isValid, err = sb.VerifyPieceInclusionProof(1024, maxPieceSize, commP, status.CommD, status.Pieces[0].InclusionProof)
require.NoError(t, err)
require.True(t, isValid)
// generate a PoSt
proofs, faults, err := sb.GeneratePoSt(ptr, [][32]byte{status.CommR}, [32]byte{})
require.NoError(t, err)
// verify the PoSt
isValid, err = sb.VerifyPoSt(1024, [][32]byte{status.CommR}, [32]byte{}, proofs, faults)
require.True(t, isValid)
// unseal the sector and retrieve the client's piece, verifying that the
// retrieved bytes match what we originally wrote to the staged sector
unsealedPieceBytes, err := sb.ReadPieceFromSealedSector(ptr, "snoqualmie")
require.Equal(t, pieceBytes, unsealedPieceBytes)
}
func pollForSectorSealingStatus(ptr unsafe.Pointer, sectorID uint64, sealStatusCode uint8, timeout time.Duration) (status sb.SectorSealingStatus, retErr error) {
timeoutCh := time.After(timeout)
tick := time.Tick(5 * time.Second)
for {
select {
case <-timeoutCh:
retErr = errors.New("timed out waiting for sector to finish sealing")
return
case <-tick:
sealingStatus, err := sb.GetSectorSealingStatusByID(ptr, sectorID)
if err != nil {
retErr = err
return
}
if sealingStatus.SealStatusCode == sealStatusCode {
status = sealingStatus
return
}
}
}
}
func sectorIdAsBytes(sectorID uint64) [31]byte {
slice := make([]byte, 31)
binary.LittleEndian.PutUint64(slice, sectorID)
var sectorIDAsBytes [31]byte
copy(sectorIDAsBytes[:], slice)
return sectorIDAsBytes
}
func requireTempFilePath(t *testing.T, fileContentsReader io.Reader) string {
file, err := ioutil.TempFile("", "")
require.NoError(t, err)
_, err = io.Copy(file, fileContentsReader)
require.NoError(t, err)
return file.Name()
}
func requireTempDirPath(t *testing.T) string {
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
return dir
}
go.sum 0 → 100644
View file @ d68ecf1b
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
build.sh → install-rust-fil-sector-builder
View file @ d68ecf1b
...@@ -11,10 +11,11 @@ git submodule update --init --recursive $subm_dir ...@@ -11,10 +11,11 @@ git submodule update --init --recursive $subm_dir
if download_release_tarball tarball_path "${subm_dir}"; then if download_release_tarball tarball_path "${subm_dir}"; then
tmp_dir=$(mktemp -d) tmp_dir=$(mktemp -d)
tar -C $tmp_dir -xzf $tarball_path tar -C "$tmp_dir" -xzf "$tarball_path"
cp -R "${tmp_dir}/include" . cp "${tmp_dir}/include/sector_builder_ffi.h" .
cp -R "${tmp_dir}/lib" . cp "${tmp_dir}/lib/libsector_builder_ffi.a" .
cp "${tmp_dir}/lib/pkgconfig/sector_builder_ffi.pc" .
else else
echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build" echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build"
build_from_source "${subm_dir}" build_from_source "${subm_dir}"
...@@ -22,7 +23,7 @@ else ...@@ -22,7 +23,7 @@ else
mkdir -p include mkdir -p include
mkdir -p lib/pkgconfig mkdir -p lib/pkgconfig
find "${subm_dir}" -type f -name sector_builder_ffi.h -exec mv -- "{}" include/ \; find "${subm_dir}" -type f -name sector_builder_ffi.h -exec cp -- "{}" . \;
find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" lib/ \; find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" . \;
find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" pkgconfig/ \; find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" . \;
fi fi
\ No newline at end of file
rust-fil-sector-builder @ 1f1b7244
Compare 394b5b1b...1f1b7244
Subproject commit 394b5b1bd87efbee6489a7d22dfb3b1dba697af0 Subproject commit 1f1b7244fd2bb82cf4fe12106a1fc012f9d42998
binding/cgo_transforms.go → transforms.go
View file @ d68ecf1b
package sectorbuilder package go_sectorbuilder
import ( import (
"unsafe" "unsafe"
) )
// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi // #cgo LDFLAGS: ${SRCDIR}/libsector_builder_ffi.a
// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc // #cgo pkg-config: ${SRCDIR}/sector_builder_ffi.pc
// #include "../include/sector_builder_ffi.h" // #include "./sector_builder_ffi.h"
import "C" import "C"
// SingleProofPartitionProofLen denotes the number of bytes in a proof generated // SingleProofPartitionProofLen denotes the number of bytes in a proof generated
...@@ -81,9 +81,15 @@ func goPieceMetadata(src *C.sector_builder_ffi_FFIPieceMetadata, size C.size_t) ...@@ -81,9 +81,15 @@ func goPieceMetadata(src *C.sector_builder_ffi_FFIPieceMetadata, size C.size_t)
ptrs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(unsafe.Pointer(src))[:size:size] ptrs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(unsafe.Pointer(src))[:size:size]
for i := 0; i < int(size); i++ { for i := 0; i < int(size); i++ {
commPSlice := goBytes(&ptrs[i].comm_p[0], CommitmentBytesLen)
var commP [CommitmentBytesLen]byte
copy(commP[:], commPSlice)
ps[i] = PieceMetadata{ ps[i] = PieceMetadata{
Key: C.GoString(ptrs[i].piece_key), Key: C.GoString(ptrs[i].piece_key),
Size: uint64(ptrs[i].num_bytes), Size: uint64(ptrs[i].num_bytes),
CommP: commP,
InclusionProof: goBytes(ptrs[i].piece_inclusion_proof_ptr, ptrs[i].piece_inclusion_proof_len),
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment