Skip to content

GitLab

"git:/anongit.freedesktop.org/drm/drm-firmware" did not exist on "b79d2396bc630bfd9b4058459d3e82d7c3428599"
Unverified Commit 181742fd authored by Erin Swenson-Healey's avatar Erin Swenson-Healey Committed by GitHub
Browse files

feat(api): expose do-it-yourself sealing API (#46) 

* feat(api): expose do-it-yourself sealing API

* wip

* ci(parallelism): run tests sequentially

* feat(import): import sealed sector API + test

* fix(dealloc): deallocate the things

* submodule bump

* submodule point to master
parent 2886db51
Hide whitespace changes
Inline Side-by-side
.circleci/config.yml
View file @ 181742fd
......@@ -111,7 +111,7 @@ commands:
steps:
- run:
name: Test project
command: RUST_LOG=info go test
command: RUST_LOG=info go test -p 1
restore_parameter_cache:
steps:
- restore_cache:
......
bindings.go
View file @ 181742fd
......@@ -117,6 +117,17 @@ type SealPreCommitOutput struct {
Ticket SealTicket
}
// RawSealPreCommitOutput is used to acquire a seed from the chain for the
// second step of Interactive PoRep. The PersistentAux is not expected to appear
// on-chain, but is needed for committing. This struct is useful for standalone
// (e.g. no sector builder) sealing.
type RawSealPreCommitOutput struct {
CommC [CommitmentBytesLen]byte
CommD [CommitmentBytesLen]byte
CommR [CommitmentBytesLen]byte
CommRLast [CommitmentBytesLen]byte
}
// SealCommitOutput is produced by the second step of Interactive PoRep.
type SealCommitOutput struct {
SectorID uint64
......@@ -188,7 +199,7 @@ func VerifySeal(
defer C.free(seedCBytes)
// a mutable pointer to a VerifySealResponse C-struct
resPtr := C.sector_builder_ffi_verify_seal(
resPtr := C.sector_builder_ffi_reexported_verify_seal(
C.uint64_t(sectorSize),
(*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
......@@ -199,7 +210,7 @@ func VerifySeal(
(*C.uint8_t)(proofCBytes),
C.size_t(len(proof)),
)
defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr)
defer C.sector_builder_ffi_reexported_destroy_verify_seal_response(resPtr)
if resPtr.status_code != 0 {
return false, errors.New(C.GoString(resPtr.error_msg))
......@@ -252,7 +263,7 @@ func VerifyPoSt(
defer C.free(unsafe.Pointer(faultsPtr))
// a mutable pointer to a VerifyPoStResponse C-struct
resPtr := C.sector_builder_ffi_verify_post(
resPtr := C.sector_builder_ffi_reexported_verify_post(
C.uint64_t(sectorSize),
(*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
sectorIdsPtr,
......@@ -264,7 +275,7 @@ func VerifyPoSt(
(*C.uint8_t)(proofCBytes),
C.size_t(len(proof)),
)
defer C.sector_builder_ffi_destroy_verify_post_response(resPtr)
defer C.sector_builder_ffi_reexported_destroy_verify_post_response(resPtr)
if resPtr.status_code != 0 {
return false, errors.New(C.GoString(resPtr.error_msg))
......@@ -279,7 +290,7 @@ func VerifyPoSt(
func GetMaxUserBytesPerStagedSector(sectorSize uint64) uint64 {
defer elapsed("GetMaxUserBytesPerStagedSector")()
return uint64(C.sector_builder_ffi_get_max_user_bytes_per_staged_sector(C.uint64_t(sectorSize)))
return uint64(C.sector_builder_ffi_reexported_get_max_user_bytes_per_staged_sector(C.uint64_t(sectorSize)))
}
// InitSectorBuilder allocates and returns a pointer to a sector builder.
......@@ -312,13 +323,8 @@ func InitSectorBuilder(
cSectorCacheRootDir := C.CString(sectorCacheRootDir)
defer C.free(unsafe.Pointer(cSectorCacheRootDir))
class, err := cSectorClass(sectorSize, poRepProofPartitions)
if err != nil {
return nil, errors.Wrap(err, "failed to get sector class")
}
resPtr := C.sector_builder_ffi_init_sector_builder(
class,
cSectorClass(sectorSize, poRepProofPartitions),
C.uint64_t(lastUsedSectorID),
cMetadataDir,
(*[32]C.uint8_t)(proverIDCBytes),
......@@ -449,7 +455,7 @@ func SealPreCommit(sectorBuilderPtr unsafe.Pointer, sectorID uint64, ticket Seal
return SealPreCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
}
out, err := goSealPreCommitOutput(resPtr)
out, err := goSectorBuilderSealPreCommitOutput(resPtr)
if err != nil {
return SealPreCommitOutput{}, err
}
......@@ -499,7 +505,7 @@ func SealCommit(sectorBuilderPtr unsafe.Pointer, sectorID uint64, seed SealSeed)
return SealCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
}
out, err := goSealCommitOutput(resPtr)
out, err := goSectorBuilderSealCommitOutput(resPtr)
if err != nil {
return SealCommitOutput{}, err
}
......@@ -677,6 +683,106 @@ func GeneratePoSt(
return goBytes(resPtr.proof_ptr, resPtr.proof_len), nil
}
// AcquireSectorId returns a sector ID which can be used by out-of-band sealing.
func AcquireSectorId(
sectorBuilderPtr unsafe.Pointer,
) (uint64, error) {
defer elapsed("AcquireSectorId")()
resPtr := C.sector_builder_ffi_acquire_sector_id(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
)
defer C.sector_builder_ffi_destroy_acquire_sector_id_response(resPtr)
if resPtr.status_code != 0 {
return 0, errors.New(C.GoString(resPtr.error_msg))
}
return uint64(resPtr.sector_id), nil
}
// ImportSealedSector
func ImportSealedSector(
sectorBuilderPtr unsafe.Pointer,
sectorID uint64,
sectorCacheDirPath string,
sealedSectorPath string,
ticket SealTicket,
seed SealSeed,
commR [CommitmentBytesLen]byte,
commD [CommitmentBytesLen]byte,
commC [CommitmentBytesLen]byte,
commRLast [CommitmentBytesLen]byte,
proof []byte,
pieces []PieceMetadata,
) error {
defer elapsed("ImportSealedSector")()
cSectorCacheDirPath := C.CString(sectorCacheDirPath)
defer C.free(unsafe.Pointer(cSectorCacheDirPath))
cSealedSectorPath := C.CString(sealedSectorPath)
defer C.free(unsafe.Pointer(cSealedSectorPath))
cTicketBytes := C.CBytes(ticket.TicketBytes[:])
defer C.free(cTicketBytes)
cSealTicket := C.sector_builder_ffi_FFISealTicket{
block_height: C.uint64_t(ticket.BlockHeight),
ticket_bytes: *(*[32]C.uint8_t)(cTicketBytes),
}
cSeedBytes := C.CBytes(seed.TicketBytes[:])
defer C.free(cSeedBytes)
cSealSeed := C.sector_builder_ffi_FFISealSeed{
block_height: C.uint64_t(seed.BlockHeight),
ticket_bytes: *(*[32]C.uint8_t)(cSeedBytes),
}
commDCBytes := C.CBytes(commD[:])
defer C.free(commDCBytes)
commRCBytes := C.CBytes(commR[:])
defer C.free(commRCBytes)
commCCBytes := C.CBytes(commC[:])
defer C.free(commCCBytes)
commRLastCBytes := C.CBytes(commRLast[:])
defer C.free(commRLastCBytes)
proofCBytes := C.CBytes(proof[:])
defer C.free(proofCBytes)
piecesPtr, piecesLen := cPieceMetadata(pieces)
defer C.free(unsafe.Pointer(piecesPtr))
resPtr := C.sector_builder_ffi_import_sealed_sector(
(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
C.uint64_t(sectorID),
cSectorCacheDirPath,
cSealedSectorPath,
cSealTicket,
cSealSeed,
(*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commCCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commRLastCBytes),
(*C.uint8_t)(proofCBytes),
C.size_t(len(proof)),
piecesPtr,
piecesLen,
)
defer C.sector_builder_ffi_destroy_import_sealed_sector_response(resPtr)
if resPtr.status_code != 0 {
return errors.New(C.GoString(resPtr.error_msg))
}
return nil
}
// GeneratePieceCommitment produces a piece commitment for the provided data
// stored at a given path.
func GeneratePieceCommitment(piecePath string, pieceSize uint64) ([CommitmentBytesLen]byte, error) {
......@@ -694,18 +800,14 @@ func GenerateDataCommitment(sectorSize uint64, pieces []PublicPieceInfo) ([Commi
cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
defer C.free(unsafe.Pointer(cPiecesPtr))
resPtr := C.sector_builder_ffi_generate_data_commitment(C.uint64_t(sectorSize), (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr), cPiecesLen)
defer C.sector_builder_ffi_destroy_generate_data_commitment_response(resPtr)
resPtr := C.sector_builder_ffi_reexported_generate_data_commitment(C.uint64_t(sectorSize), (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr), cPiecesLen)
defer C.sector_builder_ffi_reexported_destroy_generate_data_commitment_response(resPtr)
if resPtr.status_code != 0 {
return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
}
commDSlice := goBytes(&resPtr.comm_d[0], CommitmentBytesLen)
var commitment [CommitmentBytesLen]byte
copy(commitment[:], commDSlice)
return commitment, nil
return goCommitment(&resPtr.comm_d[0]), nil
}
// GeneratePieceCommitmentFromFile produces a piece commitment for the provided data
......@@ -713,8 +815,8 @@ func GenerateDataCommitment(sectorSize uint64, pieces []PublicPieceInfo) ([Commi
func GeneratePieceCommitmentFromFile(pieceFile *os.File, pieceSize uint64) (commP [CommitmentBytesLen]byte, err error) {
pieceFd := pieceFile.Fd()
resPtr := C.sector_builder_ffi_generate_piece_commitment(C.int(pieceFd), C.uint64_t(pieceSize))
defer C.sector_builder_ffi_destroy_generate_piece_commitment_response(resPtr)
resPtr := C.sector_builder_ffi_reexported_generate_piece_commitment(C.int(pieceFd), C.uint64_t(pieceSize))
defer C.sector_builder_ffi_reexported_destroy_generate_piece_commitment_response(resPtr)
// Make sure our filedescriptor stays alive, stayin alive
runtime.KeepAlive(pieceFile)
......@@ -723,11 +825,216 @@ func GeneratePieceCommitmentFromFile(pieceFile *os.File, pieceSize uint64) (comm
return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
}
commPSlice := goBytes(&resPtr.comm_p[0], CommitmentBytesLen)
var commitment [CommitmentBytesLen]byte
copy(commitment[:], commPSlice)
return goCommitment(&resPtr.comm_p[0]), nil
}
// StandaloneWriteWithAlignment
func StandaloneWriteWithAlignment(
pieceFile *os.File,
pieceBytes uint64,
stagedSectorFile *os.File,
existingPieceSizes []uint64,
) (leftAlignment, total uint64, commP [CommitmentBytesLen]byte, retErr error) {
defer elapsed("StandaloneWriteWithAlignment")()
pieceFd := pieceFile.Fd()
runtime.KeepAlive(pieceFile)
stagedSectorFd := stagedSectorFile.Fd()
runtime.KeepAlive(stagedSectorFile)
ptr, len := cUint64s(existingPieceSizes)
defer C.free(unsafe.Pointer(ptr))
resPtr := C.sector_builder_ffi_reexported_write_with_alignment(
C.int(pieceFd),
C.uint64_t(pieceBytes),
C.int(stagedSectorFd),
ptr,
len,
)
defer C.sector_builder_ffi_reexported_destroy_write_with_alignment_response(resPtr)
if resPtr.status_code != 0 {
return 0, 0, [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
}
return uint64(resPtr.left_alignment_unpadded), uint64(resPtr.total_write_unpadded), goCommitment(&resPtr.comm_p[0]), nil
}
// StandaloneWriteWithoutAlignment
func StandaloneWriteWithoutAlignment(
pieceFile *os.File,
pieceBytes uint64,
stagedSectorFile *os.File,
) (uint64, [CommitmentBytesLen]byte, error) {
defer elapsed("StandaloneWriteWithoutAlignment")()
pieceFd := pieceFile.Fd()
runtime.KeepAlive(pieceFile)
stagedSectorFd := stagedSectorFile.Fd()
runtime.KeepAlive(stagedSectorFile)
resPtr := C.sector_builder_ffi_reexported_write_without_alignment(
C.int(pieceFd),
C.uint64_t(pieceBytes),
C.int(stagedSectorFd),
)
defer C.sector_builder_ffi_reexported_destroy_write_without_alignment_response(resPtr)
if resPtr.status_code != 0 {
return 0, [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
}
return uint64(resPtr.total_write_unpadded), goCommitment(&resPtr.comm_p[0]), nil
}
// StandaloneSealPreCommit
func StandaloneSealPreCommit(
sectorSize uint64,
poRepProofPartitions uint8,
cacheDirPath string,
stagedSectorPath string,
sealedSectorPath string,
sectorID uint64,
proverID [CommitmentBytesLen]byte,
ticket [CommitmentBytesLen]byte,
pieces []PublicPieceInfo,
) (RawSealPreCommitOutput, error) {
defer elapsed("StandaloneSealPreCommit")()
cCacheDirPath := C.CString(cacheDirPath)
defer C.free(unsafe.Pointer(cCacheDirPath))
cStagedSectorPath := C.CString(stagedSectorPath)
defer C.free(unsafe.Pointer(cStagedSectorPath))
cSealedSectorPath := C.CString(sealedSectorPath)
defer C.free(unsafe.Pointer(cSealedSectorPath))
proverIDCBytes := C.CBytes(proverID[:])
defer C.free(proverIDCBytes)
ticketCBytes := C.CBytes(ticket[:])
defer C.free(ticketCBytes)
cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
defer C.free(unsafe.Pointer(cPiecesPtr))
resPtr := C.sector_builder_ffi_reexported_seal_pre_commit(
cSectorClass(sectorSize, poRepProofPartitions),
cCacheDirPath,
cStagedSectorPath,
cSealedSectorPath,
C.uint64_t(sectorID),
(*[32]C.uint8_t)(proverIDCBytes),
(*[32]C.uint8_t)(ticketCBytes),
(*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr),
cPiecesLen,
)
defer C.sector_builder_ffi_reexported_destroy_seal_pre_commit_response(resPtr)
if resPtr.status_code != 0 {
return RawSealPreCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
}
return goRawSealPreCommitOutput(resPtr.seal_pre_commit_output), nil
}
// StandaloneSealCommit
func StandaloneSealCommit(
sectorSize uint64,
poRepProofPartitions uint8,
cacheDirPath string,
sectorID uint64,
proverID [CommitmentBytesLen]byte,
ticket [CommitmentBytesLen]byte,
seed [CommitmentBytesLen]byte,
pieces []PublicPieceInfo,
rspco RawSealPreCommitOutput,
) ([]byte, error) {
defer elapsed("StandaloneSealCommit")()
cCacheDirPath := C.CString(cacheDirPath)
defer C.free(unsafe.Pointer(cCacheDirPath))
proverIDCBytes := C.CBytes(proverID[:])
defer C.free(proverIDCBytes)
ticketCBytes := C.CBytes(ticket[:])
defer C.free(ticketCBytes)
seedCBytes := C.CBytes(seed[:])
defer C.free(seedCBytes)
cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
defer C.free(unsafe.Pointer(cPiecesPtr))
resPtr := C.sector_builder_ffi_reexported_seal_commit(
cSectorClass(sectorSize, poRepProofPartitions),
cCacheDirPath,
C.uint64_t(sectorID),
(*[32]C.uint8_t)(proverIDCBytes),
(*[32]C.uint8_t)(ticketCBytes),
(*[32]C.uint8_t)(seedCBytes),
(*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr),
cPiecesLen,
cSealPreCommitOutput(rspco),
)
defer C.sector_builder_ffi_reexported_destroy_seal_commit_response(resPtr)
if resPtr.status_code != 0 {
return nil, errors.New(C.GoString(resPtr.error_msg))
}
return C.GoBytes(unsafe.Pointer(resPtr.proof_ptr), C.int(resPtr.proof_len)), nil
}
// StandaloneUnseal
func StandaloneUnseal(
sectorSize uint64,
poRepProofPartitions uint8,
sealedSectorPath string,
unsealOutputPath string,
sectorID uint64,
proverID [CommitmentBytesLen]byte,
ticket [CommitmentBytesLen]byte,
commD [CommitmentBytesLen]byte,
) error {
defer elapsed("StandaloneUnseal")()
cSealedSectorPath := C.CString(sealedSectorPath)
defer C.free(unsafe.Pointer(cSealedSectorPath))
cUnsealOutputPath := C.CString(unsealOutputPath)
defer C.free(unsafe.Pointer(cUnsealOutputPath))
proverIDCBytes := C.CBytes(proverID[:])
defer C.free(proverIDCBytes)
ticketCBytes := C.CBytes(ticket[:])
defer C.free(ticketCBytes)
commDCBytes := C.CBytes(commD[:])
defer C.free(commDCBytes)
resPtr := C.sector_builder_ffi_reexported_unseal(
cSectorClass(sectorSize, poRepProofPartitions),
cSealedSectorPath,
cUnsealOutputPath,
C.uint64_t(sectorID),
(*[CommitmentBytesLen]C.uint8_t)(proverIDCBytes),
(*[CommitmentBytesLen]C.uint8_t)(ticketCBytes),
(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
)
defer C.sector_builder_ffi_reexported_destroy_unseal_response(resPtr)
if resPtr.status_code != 0 {
return errors.New(C.GoString(resPtr.error_msg))
}
return commitment, nil
return nil
}
func getAllSealedSectors(sectorBuilderPtr unsafe.Pointer, performHealthchecks bool) ([]SealedSectorMetadata, error) {
......
bindings_test.go
View file @ 181742fd
......@@ -3,12 +3,15 @@ package go_sectorbuilder_test
import (
"bytes"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"github.com/stretchr/testify/assert"
"io"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"testing"
"time"
"unsafe"
......@@ -44,16 +47,16 @@ func TestSectorBuilderLifecycle(t *testing.T) {
proverID := [32]byte{6, 7, 8}
metadataDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(metadataDir))
defer os.RemoveAll(metadataDir)
sealedSectorDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(sealedSectorDir))
defer os.RemoveAll(sealedSectorDir)
stagedSectorDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(stagedSectorDir))
defer os.RemoveAll(stagedSectorDir)
sectorCacheRootDir := requireTempDirPath(t)
defer require.NoError(t, os.Remove(sectorCacheRootDir))
defer os.RemoveAll(sectorCacheRootDir)
ptr, err := sb.InitSectorBuilder(1024, 2, 0, metadataDir, proverID, sealedSectorDir, stagedSectorDir, sectorCacheRootDir, 1, 2)
require.NoError(t, err)
......@@ -201,6 +204,169 @@ func TestSectorBuilderLifecycle(t *testing.T) {
require.Equal(t, pieceBytes, unsealedPieceBytes)
}
func TestStandaloneSealing(t *testing.T) {
sectorSize := uint64(1024)
poRepProofPartitions := uint8(2)
ticket := sb.SealTicket{
BlockHeight: 0,
TicketBytes: [32]byte{5, 4, 2},
}
seed := sb.SealSeed{
BlockHeight: 50,
TicketBytes: [32]byte{7, 4, 2},
}
proverID := [32]byte{6, 7, 8}
// initialize a sector builder
metadataDir := requireTempDirPath(t)
defer os.RemoveAll(metadataDir)
sealedSectorsDir := requireTempDirPath(t)
defer os.RemoveAll(sealedSectorsDir)
stagedSectorsDir := requireTempDirPath(t)
defer os.RemoveAll(stagedSectorsDir)
sectorCacheRootDir := requireTempDirPath(t)
defer os.RemoveAll(sectorCacheRootDir)
ptr, err := sb.InitSectorBuilder(1024, 2, 0, metadataDir, proverID, sealedSectorsDir, stagedSectorsDir, sectorCacheRootDir, 1, 1)
require.NoError(t, err)
defer sb.DestroySectorBuilder(ptr)
sectorID, err := sb.AcquireSectorId(ptr)
require.NoError(t, err)
sectorCacheDirPath := requireTempDirPath(t)
defer os.RemoveAll(sectorCacheDirPath)
stagedSectorFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
defer stagedSectorFile.Close()
sealedSectorFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
defer sealedSectorFile.Close()
unsealOutputFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
defer unsealOutputFile.Close()
// some rando bytes
someBytes := make([]byte, 1016)
_, err = io.ReadFull(rand.Reader, someBytes)
require.NoError(t, err)
// write first piece
require.NoError(t, err)
pieceFileA := requireTempFile(t, bytes.NewReader(someBytes[0:127]), 127)
commPA, err := sb.GeneratePieceCommitmentFromFile(pieceFileA, 127)
require.NoError(t, err)
// seek back to head (generating piece commitment moves offset)
_, err = pieceFileA.Seek(0, 0)
require.NoError(t, err)
// write the first piece using the alignment-free function
n, commP, err := sb.StandaloneWriteWithoutAlignment(pieceFileA, 127, stagedSectorFile)
require.NoError(t, err)
require.Equal(t, int(n), 127)
require.Equal(t, commP, commPA)
// write second piece + alignment
require.NoError(t, err)
pieceFileB := requireTempFile(t, bytes.NewReader(someBytes[0:508]), 508)
commPB, err := sb.GeneratePieceCommitmentFromFile(pieceFileB, 508)
require.NoError(t, err)
// seek back to head
_, err = pieceFileB.Seek(0, 0)
require.NoError(t, err)
// second piece relies on the alignment-computing version
left, tot, commP, err := sb.StandaloneWriteWithAlignment(pieceFileB, 508, stagedSectorFile, []uint64{127})
require.NoError(t, err)
require.Equal(t, int(left), 381)
require.Equal(t, int(tot), 889)
require.Equal(t, commP, commPB)
publicPieces := []sb.PublicPieceInfo{{
Size: 127,
CommP: commPA,
}, {
Size: 508,
CommP: commPB,
}}
privatePieces := make([]sb.PieceMetadata, len(publicPieces))
for i, v := range publicPieces {
privatePieces[i] = sb.PieceMetadata{
Key: hex.EncodeToString(v.CommP[:]),
Size: v.Size,
CommP: v.CommP,
}
}
// pre-commit the sector
output, err := sb.StandaloneSealPreCommit(sectorSize, poRepProofPartitions, sectorCacheDirPath, stagedSectorFile.Name(), sealedSectorFile.Name(), sectorID, proverID, ticket.TicketBytes, publicPieces)
require.NoError(t, err)
// commit the sector
proof, err := sb.StandaloneSealCommit(sectorSize, poRepProofPartitions, sectorCacheDirPath, sectorID, proverID, ticket.TicketBytes, seed.TicketBytes, publicPieces, output)
require.NoError(t, err)
// verify the 'ole proofy
isValid, err := sb.VerifySeal(sectorSize, output.CommR, output.CommD, proverID, ticket.TicketBytes, seed.TicketBytes, sectorID, proof)
require.NoError(t, err)
require.True(t, isValid, "proof wasn't valid")
// unseal and verify that things went as we planned
require.NoError(t, sb.StandaloneUnseal(sectorSize, poRepProofPartitions, sealedSectorFile.Name(), unsealOutputFile.Name(), sectorID, proverID, ticket.TicketBytes, output.CommD))
contents, err := ioutil.ReadFile(unsealOutputFile.Name())
require.NoError(t, err)
// unsealed sector includes a bunch of alignment NUL-bytes
alignment := make([]byte, 381)
// verify that we unsealed what we expected to unseal
require.Equal(t, someBytes[0:127], contents[0:127])
require.Equal(t, alignment, contents[127:508])
require.Equal(t, someBytes[0:508], contents[508:1016])
// verify that the sector builder owns no sealed sectors
var sealedSectorPaths []string
require.NoError(t, filepath.Walk(sealedSectorsDir, visit(&sealedSectorPaths)))
assert.Equal(t, 1, len(sealedSectorPaths), sealedSectorPaths)
// no sector cache dirs, either
var sectorCacheDirPaths []string
require.NoError(t, filepath.Walk(sectorCacheRootDir, visit(&sectorCacheDirPaths)))
assert.Equal(t, 1, len(sectorCacheDirPaths), sectorCacheDirPaths)
// import the sealed sector
err = sb.ImportSealedSector(ptr, sectorID, sectorCacheDirPath, sealedSectorFile.Name(), ticket, seed, output.CommR, output.CommD, output.CommC, output.CommRLast, proof, privatePieces)
require.NoError(t, err)
// it should now have a sealed sector!
var sealedSectorPathsB []string
require.NoError(t, filepath.Walk(sealedSectorsDir, visit(&sealedSectorPathsB)))
assert.Equal(t, 2, len(sealedSectorPathsB), sealedSectorPathsB)
// it should now have a cache dir, woo!
var sectorCacheDirPathsB []string
require.NoError(t, filepath.Walk(sectorCacheRootDir, visit(&sectorCacheDirPathsB)))
assert.Equal(t, 2, len(sectorCacheDirPathsB), sectorCacheDirPathsB)
// verify that it shows up in sealed sector list
metadata, err := sb.GetAllSealedSectorsWithHealth(ptr)
require.NoError(t, err)
require.Equal(t, 1, len(metadata))
require.Equal(t, output.CommD, metadata[0].CommD)
require.Equal(t, output.CommR, metadata[0].CommR)
}
func TestJsonMarshalSymmetry(t *testing.T) {
for i := 0; i < 100; i++ {
xs := make([]sb.SectorInfo, 10)
......@@ -265,7 +431,7 @@ func requireTempFile(t *testing.T, fileContentsReader io.Reader, size uint64) *o
written, err := io.Copy(file, fileContentsReader)
require.NoError(t, err)
// check that we wrote everything
require.Equal(t, uint64(written), size)
require.Equal(t, int(size), int(written))
require.NoError(t, file.Sync())
......@@ -282,3 +448,13 @@ func requireTempDirPath(t *testing.T) string {
return dir
}
func visit(paths *[]string) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
if err != nil {
panic(err)
}
*paths = append(*paths, path)
return nil
}
}
rust-fil-sector-builder @ 65d97d0d
Compare fc583ce6...65d97d0d
Subproject commit fc583ce62e9dd1a6eb672eecad7cc745333f33bb
Subproject commit 65d97d0d5aad2f94ce381c46fea3e74eab050b9c
transforms.go
View file @ 181742fd
......@@ -36,6 +36,25 @@ func cPublicPieceInfo(src []PublicPieceInfo) (*C.sector_builder_ffi_FFIPublicPie
return (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPublicPieceInfos), srcCSizeT
}
func cPieceMetadata(src []PieceMetadata) (*C.sector_builder_ffi_FFIPieceMetadata, C.size_t) {
srcCSizeT := C.size_t(len(src))
// allocate array in C heap
cPieceMetadata := C.malloc(srcCSizeT * C.sizeof_sector_builder_ffi_FFIPieceMetadata)
// create a Go slice backed by the C-array
xs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(cPieceMetadata)
for i, v := range src {
xs[i] = C.sector_builder_ffi_FFIPieceMetadata{
piece_key: C.CString(v.Key),
num_bytes: C.uint64_t(v.Size),
comm_p: *(*[32]C.uint8_t)(unsafe.Pointer(&v.CommP)),
}
}
return (*C.sector_builder_ffi_FFIPieceMetadata)(cPieceMetadata), srcCSizeT
}
func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
srcCSizeT := C.size_t(len(src))
......@@ -51,11 +70,20 @@ func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
return (*C.uint64_t)(cUint64s), srcCSizeT
}
func cSectorClass(sectorSize uint64, poRepProofPartitions uint8) (C.sector_builder_ffi_FFISectorClass, error) {
func cSectorClass(sectorSize uint64, poRepProofPartitions uint8) C.sector_builder_ffi_FFISectorClass {
return C.sector_builder_ffi_FFISectorClass{
sector_size: C.uint64_t(sectorSize),
porep_proof_partitions: C.uint8_t(poRepProofPartitions),
}, nil
}
}
func cSealPreCommitOutput(src RawSealPreCommitOutput) C.sector_builder_ffi_FFISealPreCommitOutput {
return C.sector_builder_ffi_FFISealPreCommitOutput{
comm_d: *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommD)),
comm_r: *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommR)),
p_aux_comm_c: *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommC)),
p_aux_comm_r_last: *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommRLast)),
}
}
func goBytes(src *C.uint8_t, size C.size_t) []byte {
......@@ -63,17 +91,30 @@ func goBytes(src *C.uint8_t, size C.size_t) []byte {
}
func goSealTicket(src C.sector_builder_ffi_FFISealTicket) SealTicket {
ticketBytesSlice := C.GoBytes(unsafe.Pointer(&src.ticket_bytes[0]), 32)
var ticketBytes [CommitmentBytesLen]byte
copy(ticketBytes[:], ticketBytesSlice)
return SealTicket{
TicketBytes: ticketBytes,
TicketBytes: goCommitment(&src.ticket_bytes[0]),
BlockHeight: uint64(src.block_height),
}
}
func goSealCommitOutput(src *C.sector_builder_ffi_SealCommitResponse) (SealCommitOutput, error) {
func goRawSealPreCommitOutput(src C.sector_builder_ffi_FFISealPreCommitOutput) RawSealPreCommitOutput {
return RawSealPreCommitOutput{
CommD: goCommitment(&src.comm_d[0]),
CommR: goCommitment(&src.comm_r[0]),
CommRLast: goCommitment(&src.p_aux_comm_r_last[0]),
CommC: goCommitment(&src.p_aux_comm_c[0]),
}
}
func goCommitment(src *C.uint8_t) [32]byte {
slice := C.GoBytes(unsafe.Pointer(src), 32)
var array [CommitmentBytesLen]byte
copy(array[:], slice)
return array
}
func goSectorBuilderSealCommitOutput(src *C.sector_builder_ffi_SectorBuilderSealCommitResponse) (SealCommitOutput, error) {
commDSlice := goBytes(&src.comm_d[0], CommitmentBytesLen)
var commD [CommitmentBytesLen]byte
copy(commD[:], commDSlice)
......@@ -127,7 +168,7 @@ func goResumeSealCommitOutput(src *C.sector_builder_ffi_ResumeSealCommitResponse
}, nil
}
func goSealPreCommitOutput(src *C.sector_builder_ffi_SealPreCommitResponse) (SealPreCommitOutput, error) {
func goSectorBuilderSealPreCommitOutput(src *C.sector_builder_ffi_SectorBuilderSealPreCommitResponse) (SealPreCommitOutput, error) {
commDSlice := goBytes(&src.comm_d[0], CommitmentBytesLen)
var commD [CommitmentBytesLen]byte
copy(commD[:], commDSlice)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment