feat(api): expose do-it-yourself sealing API (#46)

* feat(api): expose do-it-yourself sealing API

* wip

* ci(parallelism): run tests sequentially

* feat(import): import sealed sector API + test

* fix(dealloc): deallocate the things

* submodule bump

* submodule point to master

.circleci/config.yml

@@ -111,7 +111,7 @@ commands:
     steps:
       - run:
           name: Test project
-          command: RUST_LOG=info go test
+          command: RUST_LOG=info go test -p 1
   restore_parameter_cache:
     steps:
       - restore_cache:

bindings.go

@@ -117,6 +117,17 @@ type SealPreCommitOutput struct {
 	Ticket   SealTicket
 }
 
+// RawSealPreCommitOutput is used to acquire a seed from the chain for the
+// second step of Interactive PoRep. The PersistentAux is not expected to appear
+// on-chain, but is needed for committing. This struct is useful for standalone
+// (e.g. no sector builder) sealing.
+type RawSealPreCommitOutput struct {
+	CommC     [CommitmentBytesLen]byte
+	CommD     [CommitmentBytesLen]byte
+	CommR     [CommitmentBytesLen]byte
+	CommRLast [CommitmentBytesLen]byte
+}
+
 // SealCommitOutput is produced by the second step of Interactive PoRep.
 type SealCommitOutput struct {
 	SectorID uint64
@@ -188,7 +199,7 @@ func VerifySeal(
 	defer C.free(seedCBytes)
 
 	// a mutable pointer to a VerifySealResponse C-struct
-	resPtr := C.sector_builder_ffi_verify_seal(
+	resPtr := C.sector_builder_ffi_reexported_verify_seal(
 		C.uint64_t(sectorSize),
 		(*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
 		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
@@ -199,7 +210,7 @@ func VerifySeal(
 		(*C.uint8_t)(proofCBytes),
 		C.size_t(len(proof)),
 	)
-	defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr)
+	defer C.sector_builder_ffi_reexported_destroy_verify_seal_response(resPtr)
 
 	if resPtr.status_code != 0 {
 		return false, errors.New(C.GoString(resPtr.error_msg))
@@ -252,7 +263,7 @@ func VerifyPoSt(
 	defer C.free(unsafe.Pointer(faultsPtr))
 
 	// a mutable pointer to a VerifyPoStResponse C-struct
-	resPtr := C.sector_builder_ffi_verify_post(
+	resPtr := C.sector_builder_ffi_reexported_verify_post(
 		C.uint64_t(sectorSize),
 		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
 		sectorIdsPtr,
@@ -264,7 +275,7 @@ func VerifyPoSt(
 		(*C.uint8_t)(proofCBytes),
 		C.size_t(len(proof)),
 	)
-	defer C.sector_builder_ffi_destroy_verify_post_response(resPtr)
+	defer C.sector_builder_ffi_reexported_destroy_verify_post_response(resPtr)
 
 	if resPtr.status_code != 0 {
 		return false, errors.New(C.GoString(resPtr.error_msg))
@@ -279,7 +290,7 @@ func VerifyPoSt(
 func GetMaxUserBytesPerStagedSector(sectorSize uint64) uint64 {
 	defer elapsed("GetMaxUserBytesPerStagedSector")()
 
-	return uint64(C.sector_builder_ffi_get_max_user_bytes_per_staged_sector(C.uint64_t(sectorSize)))
+	return uint64(C.sector_builder_ffi_reexported_get_max_user_bytes_per_staged_sector(C.uint64_t(sectorSize)))
 }
 
 // InitSectorBuilder allocates and returns a pointer to a sector builder.
@@ -312,13 +323,8 @@ func InitSectorBuilder(
 	cSectorCacheRootDir := C.CString(sectorCacheRootDir)
 	defer C.free(unsafe.Pointer(cSectorCacheRootDir))
 
-	class, err := cSectorClass(sectorSize, poRepProofPartitions)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get sector class")
-	}
-
 	resPtr := C.sector_builder_ffi_init_sector_builder(
-		class,
+		cSectorClass(sectorSize, poRepProofPartitions),
 		C.uint64_t(lastUsedSectorID),
 		cMetadataDir,
 		(*[32]C.uint8_t)(proverIDCBytes),
@@ -449,7 +455,7 @@ func SealPreCommit(sectorBuilderPtr unsafe.Pointer, sectorID uint64, ticket Seal
 		return SealPreCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	out, err := goSealPreCommitOutput(resPtr)
+	out, err := goSectorBuilderSealPreCommitOutput(resPtr)
 	if err != nil {
 		return SealPreCommitOutput{}, err
 	}
@@ -499,7 +505,7 @@ func SealCommit(sectorBuilderPtr unsafe.Pointer, sectorID uint64, seed SealSeed)
 		return SealCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	out, err := goSealCommitOutput(resPtr)
+	out, err := goSectorBuilderSealCommitOutput(resPtr)
 	if err != nil {
 		return SealCommitOutput{}, err
 	}
@@ -677,6 +683,106 @@ func GeneratePoSt(
 	return goBytes(resPtr.proof_ptr, resPtr.proof_len), nil
 }
 
+// AcquireSectorId returns a sector ID which can be used by out-of-band sealing.
+func AcquireSectorId(
+	sectorBuilderPtr unsafe.Pointer,
+) (uint64, error) {
+	defer elapsed("AcquireSectorId")()
+
+	resPtr := C.sector_builder_ffi_acquire_sector_id(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+	)
+	defer C.sector_builder_ffi_destroy_acquire_sector_id_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return 0, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return uint64(resPtr.sector_id), nil
+}
+
+// ImportSealedSector
+func ImportSealedSector(
+	sectorBuilderPtr unsafe.Pointer,
+	sectorID uint64,
+	sectorCacheDirPath string,
+	sealedSectorPath string,
+	ticket SealTicket,
+	seed SealSeed,
+	commR [CommitmentBytesLen]byte,
+	commD [CommitmentBytesLen]byte,
+	commC [CommitmentBytesLen]byte,
+	commRLast [CommitmentBytesLen]byte,
+	proof []byte,
+	pieces []PieceMetadata,
+) error {
+	defer elapsed("ImportSealedSector")()
+
+	cSectorCacheDirPath := C.CString(sectorCacheDirPath)
+	defer C.free(unsafe.Pointer(cSectorCacheDirPath))
+
+	cSealedSectorPath := C.CString(sealedSectorPath)
+	defer C.free(unsafe.Pointer(cSealedSectorPath))
+
+	cTicketBytes := C.CBytes(ticket.TicketBytes[:])
+	defer C.free(cTicketBytes)
+
+	cSealTicket := C.sector_builder_ffi_FFISealTicket{
+		block_height: C.uint64_t(ticket.BlockHeight),
+		ticket_bytes: *(*[32]C.uint8_t)(cTicketBytes),
+	}
+
+	cSeedBytes := C.CBytes(seed.TicketBytes[:])
+	defer C.free(cSeedBytes)
+
+	cSealSeed := C.sector_builder_ffi_FFISealSeed{
+		block_height: C.uint64_t(seed.BlockHeight),
+		ticket_bytes: *(*[32]C.uint8_t)(cSeedBytes),
+	}
+
+	commDCBytes := C.CBytes(commD[:])
+	defer C.free(commDCBytes)
+
+	commRCBytes := C.CBytes(commR[:])
+	defer C.free(commRCBytes)
+
+	commCCBytes := C.CBytes(commC[:])
+	defer C.free(commCCBytes)
+
+	commRLastCBytes := C.CBytes(commRLast[:])
+	defer C.free(commRLastCBytes)
+
+	proofCBytes := C.CBytes(proof[:])
+	defer C.free(proofCBytes)
+
+	piecesPtr, piecesLen := cPieceMetadata(pieces)
+	defer C.free(unsafe.Pointer(piecesPtr))
+
+	resPtr := C.sector_builder_ffi_import_sealed_sector(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+		C.uint64_t(sectorID),
+		cSectorCacheDirPath,
+		cSealedSectorPath,
+		cSealTicket,
+		cSealSeed,
+		(*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commCCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commRLastCBytes),
+		(*C.uint8_t)(proofCBytes),
+		C.size_t(len(proof)),
+		piecesPtr,
+		piecesLen,
+	)
+	defer C.sector_builder_ffi_destroy_import_sealed_sector_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return nil
+}
+
 // GeneratePieceCommitment produces a piece commitment for the provided data
 // stored at a given path.
 func GeneratePieceCommitment(piecePath string, pieceSize uint64) ([CommitmentBytesLen]byte, error) {
@@ -694,18 +800,14 @@ func GenerateDataCommitment(sectorSize uint64, pieces []PublicPieceInfo) ([Commi
 	cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
 	defer C.free(unsafe.Pointer(cPiecesPtr))
 
-	resPtr := C.sector_builder_ffi_generate_data_commitment(C.uint64_t(sectorSize), (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr), cPiecesLen)
-	defer C.sector_builder_ffi_destroy_generate_data_commitment_response(resPtr)
+	resPtr := C.sector_builder_ffi_reexported_generate_data_commitment(C.uint64_t(sectorSize), (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr), cPiecesLen)
+	defer C.sector_builder_ffi_reexported_destroy_generate_data_commitment_response(resPtr)
 
 	if resPtr.status_code != 0 {
 		return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	commDSlice := goBytes(&resPtr.comm_d[0], CommitmentBytesLen)
-	var commitment [CommitmentBytesLen]byte
-	copy(commitment[:], commDSlice)
-
-	return commitment, nil
+	return goCommitment(&resPtr.comm_d[0]), nil
 }
 
 // GeneratePieceCommitmentFromFile produces a piece commitment for the provided data
@@ -713,8 +815,8 @@ func GenerateDataCommitment(sectorSize uint64, pieces []PublicPieceInfo) ([Commi
 func GeneratePieceCommitmentFromFile(pieceFile *os.File, pieceSize uint64) (commP [CommitmentBytesLen]byte, err error) {
 	pieceFd := pieceFile.Fd()
 
-	resPtr := C.sector_builder_ffi_generate_piece_commitment(C.int(pieceFd), C.uint64_t(pieceSize))
-	defer C.sector_builder_ffi_destroy_generate_piece_commitment_response(resPtr)
+	resPtr := C.sector_builder_ffi_reexported_generate_piece_commitment(C.int(pieceFd), C.uint64_t(pieceSize))
+	defer C.sector_builder_ffi_reexported_destroy_generate_piece_commitment_response(resPtr)
 
 	// Make sure our filedescriptor stays alive, stayin alive
 	runtime.KeepAlive(pieceFile)
@@ -723,11 +825,216 @@ func GeneratePieceCommitmentFromFile(pieceFile *os.File, pieceSize uint64) (comm
 		return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	commPSlice := goBytes(&resPtr.comm_p[0], CommitmentBytesLen)
-	var commitment [CommitmentBytesLen]byte
-	copy(commitment[:], commPSlice)
+	return goCommitment(&resPtr.comm_p[0]), nil
+}
+
+// StandaloneWriteWithAlignment
+func StandaloneWriteWithAlignment(
+	pieceFile *os.File,
+	pieceBytes uint64,
+	stagedSectorFile *os.File,
+	existingPieceSizes []uint64,
+) (leftAlignment, total uint64, commP [CommitmentBytesLen]byte, retErr error) {
+	defer elapsed("StandaloneWriteWithAlignment")()
+
+	pieceFd := pieceFile.Fd()
+	runtime.KeepAlive(pieceFile)
+
+	stagedSectorFd := stagedSectorFile.Fd()
+	runtime.KeepAlive(stagedSectorFile)
+
+	ptr, len := cUint64s(existingPieceSizes)
+	defer C.free(unsafe.Pointer(ptr))
+
+	resPtr := C.sector_builder_ffi_reexported_write_with_alignment(
+		C.int(pieceFd),
+		C.uint64_t(pieceBytes),
+		C.int(stagedSectorFd),
+		ptr,
+		len,
+	)
+	defer C.sector_builder_ffi_reexported_destroy_write_with_alignment_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return 0, 0, [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return uint64(resPtr.left_alignment_unpadded), uint64(resPtr.total_write_unpadded), goCommitment(&resPtr.comm_p[0]), nil
+}
+
+// StandaloneWriteWithoutAlignment
+func StandaloneWriteWithoutAlignment(
+	pieceFile *os.File,
+	pieceBytes uint64,
+	stagedSectorFile *os.File,
+) (uint64, [CommitmentBytesLen]byte, error) {
+	defer elapsed("StandaloneWriteWithoutAlignment")()
+
+	pieceFd := pieceFile.Fd()
+	runtime.KeepAlive(pieceFile)
+
+	stagedSectorFd := stagedSectorFile.Fd()
+	runtime.KeepAlive(stagedSectorFile)
+
+	resPtr := C.sector_builder_ffi_reexported_write_without_alignment(
+		C.int(pieceFd),
+		C.uint64_t(pieceBytes),
+		C.int(stagedSectorFd),
+	)
+	defer C.sector_builder_ffi_reexported_destroy_write_without_alignment_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return 0, [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return uint64(resPtr.total_write_unpadded), goCommitment(&resPtr.comm_p[0]), nil
+}
+
+// StandaloneSealPreCommit
+func StandaloneSealPreCommit(
+	sectorSize uint64,
+	poRepProofPartitions uint8,
+	cacheDirPath string,
+	stagedSectorPath string,
+	sealedSectorPath string,
+	sectorID uint64,
+	proverID [CommitmentBytesLen]byte,
+	ticket [CommitmentBytesLen]byte,
+	pieces []PublicPieceInfo,
+) (RawSealPreCommitOutput, error) {
+	defer elapsed("StandaloneSealPreCommit")()
+
+	cCacheDirPath := C.CString(cacheDirPath)
+	defer C.free(unsafe.Pointer(cCacheDirPath))
+
+	cStagedSectorPath := C.CString(stagedSectorPath)
+	defer C.free(unsafe.Pointer(cStagedSectorPath))
+
+	cSealedSectorPath := C.CString(sealedSectorPath)
+	defer C.free(unsafe.Pointer(cSealedSectorPath))
+
+	proverIDCBytes := C.CBytes(proverID[:])
+	defer C.free(proverIDCBytes)
+
+	ticketCBytes := C.CBytes(ticket[:])
+	defer C.free(ticketCBytes)
+
+	cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
+	defer C.free(unsafe.Pointer(cPiecesPtr))
+
+	resPtr := C.sector_builder_ffi_reexported_seal_pre_commit(
+		cSectorClass(sectorSize, poRepProofPartitions),
+		cCacheDirPath,
+		cStagedSectorPath,
+		cSealedSectorPath,
+		C.uint64_t(sectorID),
+		(*[32]C.uint8_t)(proverIDCBytes),
+		(*[32]C.uint8_t)(ticketCBytes),
+		(*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr),
+		cPiecesLen,
+	)
+	defer C.sector_builder_ffi_reexported_destroy_seal_pre_commit_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return RawSealPreCommitOutput{}, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return goRawSealPreCommitOutput(resPtr.seal_pre_commit_output), nil
+}
+
+// StandaloneSealCommit
+func StandaloneSealCommit(
+	sectorSize uint64,
+	poRepProofPartitions uint8,
+	cacheDirPath string,
+	sectorID uint64,
+	proverID [CommitmentBytesLen]byte,
+	ticket [CommitmentBytesLen]byte,
+	seed [CommitmentBytesLen]byte,
+	pieces []PublicPieceInfo,
+	rspco RawSealPreCommitOutput,
+) ([]byte, error) {
+	defer elapsed("StandaloneSealCommit")()
+
+	cCacheDirPath := C.CString(cacheDirPath)
+	defer C.free(unsafe.Pointer(cCacheDirPath))
+
+	proverIDCBytes := C.CBytes(proverID[:])
+	defer C.free(proverIDCBytes)
+
+	ticketCBytes := C.CBytes(ticket[:])
+	defer C.free(ticketCBytes)
+
+	seedCBytes := C.CBytes(seed[:])
+	defer C.free(seedCBytes)
+
+	cPiecesPtr, cPiecesLen := cPublicPieceInfo(pieces)
+	defer C.free(unsafe.Pointer(cPiecesPtr))
+
+	resPtr := C.sector_builder_ffi_reexported_seal_commit(
+		cSectorClass(sectorSize, poRepProofPartitions),
+		cCacheDirPath,
+		C.uint64_t(sectorID),
+		(*[32]C.uint8_t)(proverIDCBytes),
+		(*[32]C.uint8_t)(ticketCBytes),
+		(*[32]C.uint8_t)(seedCBytes),
+		(*C.sector_builder_ffi_FFIPublicPieceInfo)(cPiecesPtr),
+		cPiecesLen,
+		cSealPreCommitOutput(rspco),
+	)
+	defer C.sector_builder_ffi_reexported_destroy_seal_commit_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return nil, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return C.GoBytes(unsafe.Pointer(resPtr.proof_ptr), C.int(resPtr.proof_len)), nil
+}
+
+// StandaloneUnseal
+func StandaloneUnseal(
+	sectorSize uint64,
+	poRepProofPartitions uint8,
+	sealedSectorPath string,
+	unsealOutputPath string,
+	sectorID uint64,
+	proverID [CommitmentBytesLen]byte,
+	ticket [CommitmentBytesLen]byte,
+	commD [CommitmentBytesLen]byte,
+) error {
+	defer elapsed("StandaloneUnseal")()
+
+	cSealedSectorPath := C.CString(sealedSectorPath)
+	defer C.free(unsafe.Pointer(cSealedSectorPath))
+
+	cUnsealOutputPath := C.CString(unsealOutputPath)
+	defer C.free(unsafe.Pointer(cUnsealOutputPath))
+
+	proverIDCBytes := C.CBytes(proverID[:])
+	defer C.free(proverIDCBytes)
+
+	ticketCBytes := C.CBytes(ticket[:])
+	defer C.free(ticketCBytes)
+
+	commDCBytes := C.CBytes(commD[:])
+	defer C.free(commDCBytes)
+
+	resPtr := C.sector_builder_ffi_reexported_unseal(
+		cSectorClass(sectorSize, poRepProofPartitions),
+		cSealedSectorPath,
+		cUnsealOutputPath,
+		C.uint64_t(sectorID),
+		(*[CommitmentBytesLen]C.uint8_t)(proverIDCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(ticketCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
+	)
+	defer C.sector_builder_ffi_reexported_destroy_unseal_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return errors.New(C.GoString(resPtr.error_msg))
+	}
 
-	return commitment, nil
+	return nil
 }
 
 func getAllSealedSectors(sectorBuilderPtr unsafe.Pointer, performHealthchecks bool) ([]SealedSectorMetadata, error) {

bindings_test.go

@@ -3,12 +3,15 @@ package go_sectorbuilder_test
 import (
 	"bytes"
 	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"github.com/stretchr/testify/assert"
 	"io"
 	"io/ioutil"
 	"math/big"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 	"unsafe"
@@ -44,16 +47,16 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	proverID := [32]byte{6, 7, 8}
 
 	metadataDir := requireTempDirPath(t)
-	defer require.NoError(t, os.Remove(metadataDir))
+	defer os.RemoveAll(metadataDir)
 
 	sealedSectorDir := requireTempDirPath(t)
-	defer require.NoError(t, os.Remove(sealedSectorDir))
+	defer os.RemoveAll(sealedSectorDir)
 
 	stagedSectorDir := requireTempDirPath(t)
-	defer require.NoError(t, os.Remove(stagedSectorDir))
+	defer os.RemoveAll(stagedSectorDir)
 
 	sectorCacheRootDir := requireTempDirPath(t)
-	defer require.NoError(t, os.Remove(sectorCacheRootDir))
+	defer os.RemoveAll(sectorCacheRootDir)
 
 	ptr, err := sb.InitSectorBuilder(1024, 2, 0, metadataDir, proverID, sealedSectorDir, stagedSectorDir, sectorCacheRootDir, 1, 2)
 	require.NoError(t, err)
@@ -201,6 +204,169 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	require.Equal(t, pieceBytes, unsealedPieceBytes)
 }
 
+func TestStandaloneSealing(t *testing.T) {
+	sectorSize := uint64(1024)
+	poRepProofPartitions := uint8(2)
+
+	ticket := sb.SealTicket{
+		BlockHeight: 0,
+		TicketBytes: [32]byte{5, 4, 2},
+	}
+
+	seed := sb.SealSeed{
+		BlockHeight: 50,
+		TicketBytes: [32]byte{7, 4, 2},
+	}
+
+	proverID := [32]byte{6, 7, 8}
+
+	// initialize a sector builder
+	metadataDir := requireTempDirPath(t)
+	defer os.RemoveAll(metadataDir)
+
+	sealedSectorsDir := requireTempDirPath(t)
+	defer os.RemoveAll(sealedSectorsDir)
+
+	stagedSectorsDir := requireTempDirPath(t)
+	defer os.RemoveAll(stagedSectorsDir)
+
+	sectorCacheRootDir := requireTempDirPath(t)
+	defer os.RemoveAll(sectorCacheRootDir)
+
+	ptr, err := sb.InitSectorBuilder(1024, 2, 0, metadataDir, proverID, sealedSectorsDir, stagedSectorsDir, sectorCacheRootDir, 1, 1)
+	require.NoError(t, err)
+	defer sb.DestroySectorBuilder(ptr)
+
+	sectorID, err := sb.AcquireSectorId(ptr)
+	require.NoError(t, err)
+
+	sectorCacheDirPath := requireTempDirPath(t)
+	defer os.RemoveAll(sectorCacheDirPath)
+
+	stagedSectorFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
+	defer stagedSectorFile.Close()
+
+	sealedSectorFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
+	defer sealedSectorFile.Close()
+
+	unsealOutputFile := requireTempFile(t, bytes.NewReader([]byte{}), 0)
+	defer unsealOutputFile.Close()
+
+	// some rando bytes
+	someBytes := make([]byte, 1016)
+	_, err = io.ReadFull(rand.Reader, someBytes)
+	require.NoError(t, err)
+
+	// write first piece
+	require.NoError(t, err)
+	pieceFileA := requireTempFile(t, bytes.NewReader(someBytes[0:127]), 127)
+
+	commPA, err := sb.GeneratePieceCommitmentFromFile(pieceFileA, 127)
+	require.NoError(t, err)
+
+	// seek back to head (generating piece commitment moves offset)
+	_, err = pieceFileA.Seek(0, 0)
+	require.NoError(t, err)
+
+	// write the first piece using the alignment-free function
+	n, commP, err := sb.StandaloneWriteWithoutAlignment(pieceFileA, 127, stagedSectorFile)
+	require.NoError(t, err)
+	require.Equal(t, int(n), 127)
+	require.Equal(t, commP, commPA)
+
+	// write second piece + alignment
+	require.NoError(t, err)
+	pieceFileB := requireTempFile(t, bytes.NewReader(someBytes[0:508]), 508)
+
+	commPB, err := sb.GeneratePieceCommitmentFromFile(pieceFileB, 508)
+	require.NoError(t, err)
+
+	// seek back to head
+	_, err = pieceFileB.Seek(0, 0)
+	require.NoError(t, err)
+
+	// second piece relies on the alignment-computing version
+	left, tot, commP, err := sb.StandaloneWriteWithAlignment(pieceFileB, 508, stagedSectorFile, []uint64{127})
+	require.NoError(t, err)
+	require.Equal(t, int(left), 381)
+	require.Equal(t, int(tot), 889)
+	require.Equal(t, commP, commPB)
+
+	publicPieces := []sb.PublicPieceInfo{{
+		Size:  127,
+		CommP: commPA,
+	}, {
+		Size:  508,
+		CommP: commPB,
+	}}
+
+	privatePieces := make([]sb.PieceMetadata, len(publicPieces))
+	for i, v := range publicPieces {
+		privatePieces[i] = sb.PieceMetadata{
+			Key:   hex.EncodeToString(v.CommP[:]),
+			Size:  v.Size,
+			CommP: v.CommP,
+		}
+	}
+
+	// pre-commit the sector
+	output, err := sb.StandaloneSealPreCommit(sectorSize, poRepProofPartitions, sectorCacheDirPath, stagedSectorFile.Name(), sealedSectorFile.Name(), sectorID, proverID, ticket.TicketBytes, publicPieces)
+	require.NoError(t, err)
+
+	// commit the sector
+	proof, err := sb.StandaloneSealCommit(sectorSize, poRepProofPartitions, sectorCacheDirPath, sectorID, proverID, ticket.TicketBytes, seed.TicketBytes, publicPieces, output)
+	require.NoError(t, err)
+
+	// verify the 'ole proofy
+	isValid, err := sb.VerifySeal(sectorSize, output.CommR, output.CommD, proverID, ticket.TicketBytes, seed.TicketBytes, sectorID, proof)
+	require.NoError(t, err)
+	require.True(t, isValid, "proof wasn't valid")
+
+	// unseal and verify that things went as we planned
+	require.NoError(t, sb.StandaloneUnseal(sectorSize, poRepProofPartitions, sealedSectorFile.Name(), unsealOutputFile.Name(), sectorID, proverID, ticket.TicketBytes, output.CommD))
+	contents, err := ioutil.ReadFile(unsealOutputFile.Name())
+	require.NoError(t, err)
+
+	// unsealed sector includes a bunch of alignment NUL-bytes
+	alignment := make([]byte, 381)
+
+	// verify that we unsealed what we expected to unseal
+	require.Equal(t, someBytes[0:127], contents[0:127])
+	require.Equal(t, alignment, contents[127:508])
+	require.Equal(t, someBytes[0:508], contents[508:1016])
+
+	// verify that the sector builder owns no sealed sectors
+	var sealedSectorPaths []string
+	require.NoError(t, filepath.Walk(sealedSectorsDir, visit(&sealedSectorPaths)))
+	assert.Equal(t, 1, len(sealedSectorPaths), sealedSectorPaths)
+
+	// no sector cache dirs, either
+	var sectorCacheDirPaths []string
+	require.NoError(t, filepath.Walk(sectorCacheRootDir, visit(&sectorCacheDirPaths)))
+	assert.Equal(t, 1, len(sectorCacheDirPaths), sectorCacheDirPaths)
+
+	// import the sealed sector
+	err = sb.ImportSealedSector(ptr, sectorID, sectorCacheDirPath, sealedSectorFile.Name(), ticket, seed, output.CommR, output.CommD, output.CommC, output.CommRLast, proof, privatePieces)
+	require.NoError(t, err)
+
+	// it should now have a sealed sector!
+	var sealedSectorPathsB []string
+	require.NoError(t, filepath.Walk(sealedSectorsDir, visit(&sealedSectorPathsB)))
+	assert.Equal(t, 2, len(sealedSectorPathsB), sealedSectorPathsB)
+
+	// it should now have a cache dir, woo!
+	var sectorCacheDirPathsB []string
+	require.NoError(t, filepath.Walk(sectorCacheRootDir, visit(&sectorCacheDirPathsB)))
+	assert.Equal(t, 2, len(sectorCacheDirPathsB), sectorCacheDirPathsB)
+
+	// verify that it shows up in sealed sector list
+	metadata, err := sb.GetAllSealedSectorsWithHealth(ptr)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(metadata))
+	require.Equal(t, output.CommD, metadata[0].CommD)
+	require.Equal(t, output.CommR, metadata[0].CommR)
+}
+
 func TestJsonMarshalSymmetry(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		xs := make([]sb.SectorInfo, 10)
@@ -265,7 +431,7 @@ func requireTempFile(t *testing.T, fileContentsReader io.Reader, size uint64) *o
 	written, err := io.Copy(file, fileContentsReader)
 	require.NoError(t, err)
 	// check that we wrote everything
-	require.Equal(t, uint64(written), size)
+	require.Equal(t, int(size), int(written))
 
 	require.NoError(t, file.Sync())
 
@@ -282,3 +448,13 @@ func requireTempDirPath(t *testing.T) string {
 
 	return dir
 }
+
+func visit(paths *[]string) filepath.WalkFunc {
+	return func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			panic(err)
+		}
+		*paths = append(*paths, path)
+		return nil
+	}
+}

rust-fil-sector-builder @ 65d97d0d

-Subproject commit fc583ce62e9dd1a6eb672eecad7cc745333f33bb
+Subproject commit 65d97d0d5aad2f94ce381c46fea3e74eab050b9c

transforms.go

@@ -36,6 +36,25 @@ func cPublicPieceInfo(src []PublicPieceInfo) (*C.sector_builder_ffi_FFIPublicPie
 	return (*C.sector_builder_ffi_FFIPublicPieceInfo)(cPublicPieceInfos), srcCSizeT
 }
 
+func cPieceMetadata(src []PieceMetadata) (*C.sector_builder_ffi_FFIPieceMetadata, C.size_t) {
+	srcCSizeT := C.size_t(len(src))
+
+	// allocate array in C heap
+	cPieceMetadata := C.malloc(srcCSizeT * C.sizeof_sector_builder_ffi_FFIPieceMetadata)
+
+	// create a Go slice backed by the C-array
+	xs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(cPieceMetadata)
+	for i, v := range src {
+		xs[i] = C.sector_builder_ffi_FFIPieceMetadata{
+			piece_key: C.CString(v.Key),
+			num_bytes: C.uint64_t(v.Size),
+			comm_p:    *(*[32]C.uint8_t)(unsafe.Pointer(&v.CommP)),
+		}
+	}
+
+	return (*C.sector_builder_ffi_FFIPieceMetadata)(cPieceMetadata), srcCSizeT
+}
+
 func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
 	srcCSizeT := C.size_t(len(src))
 
@@ -51,11 +70,20 @@ func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
 	return (*C.uint64_t)(cUint64s), srcCSizeT
 }
 
-func cSectorClass(sectorSize uint64, poRepProofPartitions uint8) (C.sector_builder_ffi_FFISectorClass, error) {
+func cSectorClass(sectorSize uint64, poRepProofPartitions uint8) C.sector_builder_ffi_FFISectorClass {
 	return C.sector_builder_ffi_FFISectorClass{
 		sector_size:            C.uint64_t(sectorSize),
 		porep_proof_partitions: C.uint8_t(poRepProofPartitions),
-	}, nil
+	}
+}
+
+func cSealPreCommitOutput(src RawSealPreCommitOutput) C.sector_builder_ffi_FFISealPreCommitOutput {
+	return C.sector_builder_ffi_FFISealPreCommitOutput{
+		comm_d:            *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommD)),
+		comm_r:            *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommR)),
+		p_aux_comm_c:      *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommC)),
+		p_aux_comm_r_last: *(*[32]C.uint8_t)(unsafe.Pointer(&src.CommRLast)),
+	}
 }
 
 func goBytes(src *C.uint8_t, size C.size_t) []byte {
@@ -63,17 +91,30 @@ func goBytes(src *C.uint8_t, size C.size_t) []byte {
 }
 
 func goSealTicket(src C.sector_builder_ffi_FFISealTicket) SealTicket {
-	ticketBytesSlice := C.GoBytes(unsafe.Pointer(&src.ticket_bytes[0]), 32)
-	var ticketBytes [CommitmentBytesLen]byte
-	copy(ticketBytes[:], ticketBytesSlice)
-
 	return SealTicket{
-		TicketBytes: ticketBytes,
+		TicketBytes: goCommitment(&src.ticket_bytes[0]),
 		BlockHeight: uint64(src.block_height),
 	}
 }
 
-func goSealCommitOutput(src *C.sector_builder_ffi_SealCommitResponse) (SealCommitOutput, error) {
+func goRawSealPreCommitOutput(src C.sector_builder_ffi_FFISealPreCommitOutput) RawSealPreCommitOutput {
+	return RawSealPreCommitOutput{
+		CommD:     goCommitment(&src.comm_d[0]),
+		CommR:     goCommitment(&src.comm_r[0]),
+		CommRLast: goCommitment(&src.p_aux_comm_r_last[0]),
+		CommC:     goCommitment(&src.p_aux_comm_c[0]),
+	}
+}
+
+func goCommitment(src *C.uint8_t) [32]byte {
+	slice := C.GoBytes(unsafe.Pointer(src), 32)
+	var array [CommitmentBytesLen]byte
+	copy(array[:], slice)
+
+	return array
+}
+
+func goSectorBuilderSealCommitOutput(src *C.sector_builder_ffi_SectorBuilderSealCommitResponse) (SealCommitOutput, error) {
 	commDSlice := goBytes(&src.comm_d[0], CommitmentBytesLen)
 	var commD [CommitmentBytesLen]byte
 	copy(commD[:], commDSlice)
@@ -127,7 +168,7 @@ func goResumeSealCommitOutput(src *C.sector_builder_ffi_ResumeSealCommitResponse
 	}, nil
 }
 
-func goSealPreCommitOutput(src *C.sector_builder_ffi_SealPreCommitResponse) (SealPreCommitOutput, error) {
+func goSectorBuilderSealPreCommitOutput(src *C.sector_builder_ffi_SectorBuilderSealPreCommitResponse) (SealPreCommitOutput, error) {
 	commDSlice := goBytes(&src.comm_d[0], CommitmentBytesLen)
 	var commD [CommitmentBytesLen]byte
 	copy(commD[:], commDSlice)