Merge pull request #1 from filecoin-project/chores/match-go-bls-sigs-structure

chores/match go-bls-sigs structure and add tests

.gitignore

 **/*.h
 **/*.a
 **/*.pc
+.install-rust-fil-sector-builder

Makefile

+DEPS:=sector_builder_ffi.h sector_builder_ffi.pc libsector_builder_ffi.a
+
+all: $(DEPS)
+.PHONY: all
+
+
+$(DEPS): .install-rust-fil-sector-builder  ;
+
+.install-rust-fil-sector-builder: rust-fil-sector-builder
+	./install-rust-fil-sector-builder
+	@touch $@
+
+
+clean:
+	rm -rf $(DEPS) .install-rust-fil-sector-builder
+.PHONY: clean

README.md

+# go-sectorbuilder
+
+> Go bindings for the Filecoin Sector Builder
+
+## Building
+
+> make

binding/cgo_bindings.go → bindings.go

-// +build !windows
-
-package sectorbuilder
+package go_sectorbuilder
 
 import (
 	"time"
@@ -10,9 +8,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi
-// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc
-// #include "../include/sector_builder_ffi.h"
+// #cgo LDFLAGS: ${SRCDIR}/libsector_builder_ffi.a
+// #cgo pkg-config: ${SRCDIR}/sector_builder_ffi.pc
+// #include "./sector_builder_ffi.h"
 import "C"
 
 var log = logging.Logger("libsectorbuilder") // nolint: deadcode
@@ -24,6 +22,9 @@ func elapsed(what string) func() {
 	}
 }
 
+// CommitmentBytesLen is the number of bytes in a CommR, CommD, CommP, and CommRStar.
+const CommitmentBytesLen = 32
+
 // StagedSectorMetadata is a sector into which we write user piece-data before
 // sealing. Note: SectorID is unique across all staged and sealed sectors for a
 // storage miner actor.
@@ -37,9 +38,9 @@ type SectorSealingStatus struct {
 	SectorID       uint64
 	SealStatusCode uint8                    // Sealed = 0, Pending = 1, Failed = 2, Sealing = 3
 	SealErrorMsg   string                   // will be nil unless SealStatusCode == 2
-	CommD          [32]byte        // will be empty unless SealStatusCode == 0
-	CommR          [32]byte        // will be empty unless SealStatusCode == 0
-	CommRStar      [32]byte        // will be empty unless SealStatusCode == 0
+	CommD          [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
+	CommR          [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
+	CommRStar      [CommitmentBytesLen]byte // will be empty unless SealStatusCode == 0
 	Proof          []byte                   // will be empty unless SealStatusCode == 0
 	Pieces         []PieceMetadata          // will be empty unless SealStatusCode == 0
 }
@@ -49,15 +50,16 @@ type PieceMetadata struct {
 	Key            string
 	Size           uint64
 	InclusionProof []byte
+	CommP          [CommitmentBytesLen]byte
 }
 
 // VerifySeal returns true if the sealing operation from which its inputs were
 // derived was valid, and false if not.
 func VerifySeal(
 	sectorSize uint64,
-	commR [32]byte,
-	commD [32]byte,
-	commRStar [32]byte,
+	commR [CommitmentBytesLen]byte,
+	commD [CommitmentBytesLen]byte,
+	commRStar [CommitmentBytesLen]byte,
 	proverID [31]byte,
 	sectorID [31]byte,
 	proof []byte,
@@ -83,16 +85,16 @@ func VerifySeal(
 	defer C.free(sectorIDCbytes)
 
 	// a mutable pointer to a VerifySealResponse C-struct
-	resPtr := (*C.sector_builder_ffi_VerifySealResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_seal(
+	resPtr := C.sector_builder_ffi_verify_seal(
 		C.uint64_t(sectorSize),
-		(*[32]C.uint8_t)(commRCBytes),
-		(*[32]C.uint8_t)(commDCBytes),
-		(*[32]C.uint8_t)(commRStarCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commRCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commRStarCBytes),
 		(*[31]C.uint8_t)(proverIDCBytes),
 		(*[31]C.uint8_t)(sectorIDCbytes),
 		(*C.uint8_t)(proofCBytes),
 		C.size_t(len(proof)),
-	)))
+	)
 	defer C.sector_builder_ffi_destroy_verify_seal_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -106,8 +108,8 @@ func VerifySeal(
 // inputs were derived was valid, and false if not.
 func VerifyPoSt(
 	sectorSize uint64,
-	sortedCommRs [][32]byte,
-	challengeSeed [32]byte,
+	sortedCommRs [][CommitmentBytesLen]byte,
+	challengeSeed [CommitmentBytesLen]byte,
 	proofs [][]byte,
 	faults []uint64,
 ) (bool, error) {
@@ -123,9 +125,9 @@ func VerifyPoSt(
 	commRs := sortedCommRs
 
 	// flattening the byte slice makes it easier to copy into the C heap
-	flattened := make([]byte, 32*len(commRs))
+	flattened := make([]byte, CommitmentBytesLen*len(commRs))
 	for idx, commR := range commRs {
-		copy(flattened[(32*idx):(32*(1+idx))], commR[:])
+		copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
 	}
 
 	// copy bytes from Go to C heap
@@ -143,17 +145,17 @@ func VerifyPoSt(
 	defer C.free(unsafe.Pointer(faultsPtr))
 
 	// a mutable pointer to a VerifyPoStResponse C-struct
-	resPtr := (*C.sector_builder_ffi_VerifyPoStResponse)(unsafe.Pointer(C.sector_builder_ffi_verify_post(
+	resPtr := C.sector_builder_ffi_verify_post(
 		C.uint64_t(sectorSize),
 		proofPartitions,
 		(*C.uint8_t)(flattenedCommRsCBytes),
 		C.size_t(len(flattened)),
-		(*[32]C.uint8_t)(challengeSeedCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
 		proofsPtr,
 		proofsLen,
 		faultsPtr,
 		faultsSize,
-	)))
+	)
 	defer C.sector_builder_ffi_destroy_verify_post_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -203,7 +205,7 @@ func InitSectorBuilder(
 		return nil, errors.Wrap(err, "failed to get sector class")
 	}
 
-	resPtr := (*C.sector_builder_ffi_InitSectorBuilderResponse)(unsafe.Pointer(C.sector_builder_ffi_init_sector_builder(
+	resPtr := C.sector_builder_ffi_init_sector_builder(
 		class,
 		C.uint64_t(lastUsedSectorID),
 		cMetadataDir,
@@ -211,7 +213,7 @@ func InitSectorBuilder(
 		cSealedSectorDir,
 		cStagedSectorDir,
 		C.uint8_t(maxNumOpenStagedSectors),
-	)))
+	)
 	defer C.sector_builder_ffi_destroy_init_sector_builder_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -246,12 +248,12 @@ func AddPiece(
 	cPiecePath := C.CString(piecePath)
 	defer C.free(unsafe.Pointer(cPiecePath))
 
-	resPtr := (*C.sector_builder_ffi_AddPieceResponse)(unsafe.Pointer(C.sector_builder_ffi_add_piece(
+	resPtr := C.sector_builder_ffi_add_piece(
 		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
 		cPieceKey,
 		C.uint64_t(pieceSize),
 		cPiecePath,
-	)))
+	)
 	defer C.sector_builder_ffi_destroy_add_piece_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -270,7 +272,10 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string)
 	cPieceKey := C.CString(pieceKey)
 	defer C.free(unsafe.Pointer(cPieceKey))
 
-	resPtr := (*C.sector_builder_ffi_ReadPieceFromSealedSectorResponse)(unsafe.Pointer(C.sector_builder_ffi_read_piece_from_sealed_sector((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), cPieceKey)))
+	resPtr := C.sector_builder_ffi_read_piece_from_sealed_sector(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+		cPieceKey,
+	)
 	defer C.sector_builder_ffi_destroy_read_piece_from_sealed_sector_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -284,7 +289,7 @@ func ReadPieceFromSealedSector(sectorBuilderPtr unsafe.Pointer, pieceKey string)
 func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error {
 	defer elapsed("SealAllStagedSectors")()
 
-	resPtr := (*C.sector_builder_ffi_SealAllStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_seal_all_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))))
+	resPtr := C.sector_builder_ffi_seal_all_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))
 	defer C.sector_builder_ffi_destroy_seal_all_staged_sectors_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -298,14 +303,14 @@ func SealAllStagedSectors(sectorBuilderPtr unsafe.Pointer) error {
 func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadata, error) {
 	defer elapsed("GetAllStagedSectors")()
 
-	resPtr := (*C.sector_builder_ffi_GetStagedSectorsResponse)(unsafe.Pointer(C.sector_builder_ffi_get_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))))
+	resPtr := C.sector_builder_ffi_get_staged_sectors((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr))
 	defer C.sector_builder_ffi_destroy_get_staged_sectors_response(resPtr)
 
 	if resPtr.status_code != 0 {
 		return nil, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	meta, err := goStagedSectorMetadata((*C.sector_builder_ffi_FFIStagedSectorMetadata)(unsafe.Pointer(resPtr.sectors_ptr)), resPtr.sectors_len)
+	meta, err := goStagedSectorMetadata(resPtr.sectors_ptr, resPtr.sectors_len)
 	if err != nil {
 		return nil, err
 	}
@@ -319,7 +324,10 @@ func GetAllStagedSectors(sectorBuilderPtr unsafe.Pointer) ([]StagedSectorMetadat
 func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64) (SectorSealingStatus, error) {
 	defer elapsed("GetSectorSealingStatusByID")()
 
-	resPtr := (*C.sector_builder_ffi_GetSealStatusResponse)(unsafe.Pointer(C.sector_builder_ffi_get_seal_status((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), C.uint64_t(sectorID))))
+	resPtr := C.sector_builder_ffi_get_seal_status(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+		C.uint64_t(sectorID),
+	)
 	defer C.sector_builder_ffi_destroy_get_seal_status_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -333,16 +341,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
 	} else if resPtr.seal_status_code == C.Sealing {
 		return SectorSealingStatus{SealStatusCode: 3}, nil
 	} else if resPtr.seal_status_code == C.Sealed {
-		commRSlice := goBytes(&resPtr.comm_r[0], 32)
-		var commR [32]byte
+		commRSlice := goBytes(&resPtr.comm_r[0], CommitmentBytesLen)
+		var commR [CommitmentBytesLen]byte
 		copy(commR[:], commRSlice)
 
-		commDSlice := goBytes(&resPtr.comm_d[0], 32)
-		var commD [32]byte
+		commDSlice := goBytes(&resPtr.comm_d[0], CommitmentBytesLen)
+		var commD [CommitmentBytesLen]byte
 		copy(commD[:], commDSlice)
 
-		commRStarSlice := goBytes(&resPtr.comm_r_star[0], 32)
-		var commRStar [32]byte
+		commRStarSlice := goBytes(&resPtr.comm_r_star[0], CommitmentBytesLen)
+		var commRStar [CommitmentBytesLen]byte
 		copy(commRStar[:], commRStarSlice)
 
 		proof := goBytes(resPtr.proof_ptr, resPtr.proof_len)
@@ -370,16 +378,16 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
 // GeneratePoSt produces a proof-of-spacetime for the provided replica commitments.
 func GeneratePoSt(
 	sectorBuilderPtr unsafe.Pointer,
-	sortedCommRs [][32]byte,
-	challengeSeed [32]byte,
+	sortedCommRs [][CommitmentBytesLen]byte,
+	challengeSeed [CommitmentBytesLen]byte,
 ) ([][]byte, []uint64, error) {
 	defer elapsed("GeneratePoSt")()
 
 	// flattening the byte slice makes it easier to copy into the C heap
 	commRs := sortedCommRs
-	flattened := make([]byte, 32*len(commRs))
+	flattened := make([]byte, CommitmentBytesLen*len(commRs))
 	for idx, commR := range commRs {
-		copy(flattened[(32*idx):(32*(1+idx))], commR[:])
+		copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
 	}
 
 	// copy the Go byte slice into C memory
@@ -389,7 +397,12 @@ func GeneratePoSt(
 	challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0])
 
 	// a mutable pointer to a GeneratePoStResponse C-struct
-	resPtr := (*C.sector_builder_ffi_GeneratePoStResponse)(unsafe.Pointer(C.sector_builder_ffi_generate_post((*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr), (*C.uint8_t)(cflattened), C.size_t(len(flattened)), (*[32]C.uint8_t)(challengeSeedPtr))))
+	resPtr := C.sector_builder_ffi_generate_post(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+		(*C.uint8_t)(cflattened),
+		C.size_t(len(flattened)),
+		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedPtr),
+	)
 	defer C.sector_builder_ffi_destroy_generate_post_response(resPtr)
 
 	if resPtr.status_code != 0 {
@@ -403,3 +416,52 @@ func GeneratePoSt(
 
 	return proofs, goUint64s(resPtr.faults_ptr, resPtr.faults_len), nil
 }
+
+// VerifyPieceInclusionProof returns true if the piece inclusion proof is valid
+// with the given arguments.
+func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP [CommitmentBytesLen]byte, commD [CommitmentBytesLen]byte, proof []byte) (bool, error) {
+	commDCBytes := C.CBytes(commD[:])
+	defer C.free(commDCBytes)
+
+	commPCBytes := C.CBytes(commP[:])
+	defer C.free(commPCBytes)
+
+	pieceInclusionProofCBytes := C.CBytes(proof)
+	defer C.free(pieceInclusionProofCBytes)
+
+	resPtr := C.sector_builder_ffi_verify_piece_inclusion_proof(
+		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
+		(*[CommitmentBytesLen]C.uint8_t)(commPCBytes),
+		(*C.uint8_t)(pieceInclusionProofCBytes),
+		C.size_t(len(proof)),
+		C.uint64_t(pieceSize),
+		C.uint64_t(sectorSize),
+	)
+	defer C.sector_builder_ffi_destroy_verify_piece_inclusion_proof_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return false, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return bool(resPtr.is_valid), nil
+}
+
+// GeneratePieceCommitment produces a piece commitment for the provided data
+// stored at a given piece path.
+func GeneratePieceCommitment(piecePath string, pieceSize uint64) (commP [CommitmentBytesLen]byte, err error) {
+	cPiecePath := C.CString(piecePath)
+	defer C.free(unsafe.Pointer(cPiecePath))
+
+	resPtr := C.sector_builder_ffi_generate_piece_commitment(cPiecePath, C.uint64_t(pieceSize))
+	defer C.sector_builder_ffi_destroy_generate_piece_commitment_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return [CommitmentBytesLen]byte{}, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	commPSlice := goBytes(&resPtr.comm_p[0], CommitmentBytesLen)
+	var commitment [CommitmentBytesLen]byte
+	copy(commitment[:], commPSlice)
+
+	return commitment, nil
+}

bindings_test.go

+package go_sectorbuilder_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+	"unsafe"
+
+	sb "github.com/filecoin-project/go-sectorbuilder"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSectorBuilderLifecycle(t *testing.T) {
+	metadataDir := requireTempDirPath(t)
+	defer require.NoError(t, os.Remove(metadataDir))
+
+	sealedSectorDir := requireTempDirPath(t)
+	defer require.NoError(t, os.Remove(sealedSectorDir))
+
+	stagedSectorDir := requireTempDirPath(t)
+	defer require.NoError(t, os.Remove(stagedSectorDir))
+
+	ptr, err := sb.InitSectorBuilder(1024, 2, 1, 0, metadataDir, [31]byte{}, sealedSectorDir, stagedSectorDir, 1)
+	require.NoError(t, err)
+	defer sb.DestroySectorBuilder(ptr)
+
+	// compute the max user-bytes that can fit into a staged sector such that
+	// bit-padding ("preprocessing") expands the file to $SECTOR_SIZE
+	maxPieceSize := sb.GetMaxUserBytesPerStagedSector(1024)
+
+	// create a piece which consumes all available space in a new, staged
+	// sector
+	pieceBytes := make([]byte, maxPieceSize)
+	_, err = io.ReadFull(rand.Reader, pieceBytes)
+	require.NoError(t, err)
+	piecePath := requireTempFilePath(t, bytes.NewReader(pieceBytes))
+
+	// generate piece commitment
+	commP, err := sb.GeneratePieceCommitment(piecePath, maxPieceSize)
+	require.NoError(t, err)
+
+	// write a piece to a staged sector, reducing remaining space to 0 and
+	// triggering the seal job
+	sectorID, err := sb.AddPiece(ptr, "snoqualmie", maxPieceSize, piecePath)
+	require.NoError(t, err)
+
+	// block until Groth parameter cache is (lazily) hydrated and sector has
+	// been sealed (or timeout)
+	status, err := pollForSectorSealingStatus(ptr, sectorID, 0, time.Minute*30)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(status.Pieces), "expected to see the one piece we added")
+
+	// verify the seal proof
+	isValid, err := sb.VerifySeal(1024, status.CommR, status.CommD, status.CommRStar, [31]byte{}, sectorIdAsBytes(sectorID), status.Proof)
+	require.NoError(t, err)
+	require.True(t, isValid)
+
+	// verify the piece inclusion proof
+	isValid, err = sb.VerifyPieceInclusionProof(1024, maxPieceSize, commP, status.CommD, status.Pieces[0].InclusionProof)
+	require.NoError(t, err)
+	require.True(t, isValid)
+
+	// generate a PoSt
+	proofs, faults, err := sb.GeneratePoSt(ptr, [][32]byte{status.CommR}, [32]byte{})
+	require.NoError(t, err)
+
+	// verify the PoSt
+	isValid, err = sb.VerifyPoSt(1024, [][32]byte{status.CommR}, [32]byte{}, proofs, faults)
+	require.True(t, isValid)
+
+	// unseal the sector and retrieve the client's piece, verifying that the
+	// retrieved bytes match what we originally wrote to the staged sector
+	unsealedPieceBytes, err := sb.ReadPieceFromSealedSector(ptr, "snoqualmie")
+	require.Equal(t, pieceBytes, unsealedPieceBytes)
+}
+
+func pollForSectorSealingStatus(ptr unsafe.Pointer, sectorID uint64, sealStatusCode uint8, timeout time.Duration) (status sb.SectorSealingStatus, retErr error) {
+	timeoutCh := time.After(timeout)
+
+	tick := time.Tick(5 * time.Second)
+
+	for {
+		select {
+		case <-timeoutCh:
+			retErr = errors.New("timed out waiting for sector to finish sealing")
+			return
+		case <-tick:
+			sealingStatus, err := sb.GetSectorSealingStatusByID(ptr, sectorID)
+			if err != nil {
+				retErr = err
+				return
+			}
+
+			if sealingStatus.SealStatusCode == sealStatusCode {
+				status = sealingStatus
+				return
+			}
+		}
+	}
+}
+
+func sectorIdAsBytes(sectorID uint64) [31]byte {
+	slice := make([]byte, 31)
+	binary.LittleEndian.PutUint64(slice, sectorID)
+
+	var sectorIDAsBytes [31]byte
+	copy(sectorIDAsBytes[:], slice)
+
+	return sectorIDAsBytes
+}
+
+func requireTempFilePath(t *testing.T, fileContentsReader io.Reader) string {
+	file, err := ioutil.TempFile("", "")
+	require.NoError(t, err)
+
+	_, err = io.Copy(file, fileContentsReader)
+	require.NoError(t, err)
+
+	return file.Name()
+}
+
+func requireTempDirPath(t *testing.T) string {
+	dir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+
+	return dir
+}

go.mod

+module github.com/filecoin-project/go-sectorbuilder
+
+go 1.12
+
+require (
+	github.com/ipfs/go-log v0.0.1
+	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.3.0
+)

go.sum

+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
+github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

build.sh → install-rust-fil-sector-builder

@@ -11,10 +11,11 @@ git submodule update --init --recursive $subm_dir
 
 if download_release_tarball tarball_path "${subm_dir}"; then
     tmp_dir=$(mktemp -d)
-    tar -C $tmp_dir -xzf $tarball_path
+    tar -C "$tmp_dir" -xzf "$tarball_path"
 
-    cp -R "${tmp_dir}/include" .
-    cp -R "${tmp_dir}/lib" .
+    cp "${tmp_dir}/include/sector_builder_ffi.h" .
+    cp "${tmp_dir}/lib/libsector_builder_ffi.a" .
+    cp "${tmp_dir}/lib/pkgconfig/sector_builder_ffi.pc" .
 else
     echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build"
     build_from_source "${subm_dir}"
@@ -22,7 +23,7 @@ else
     mkdir -p include
     mkdir -p lib/pkgconfig
 
-    find "${subm_dir}" -type f -name sector_builder_ffi.h -exec mv -- "{}" include/ \;
-    find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" lib/ \;
-    find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" pkgconfig/ \;
+    find "${subm_dir}" -type f -name sector_builder_ffi.h -exec cp -- "{}" . \;
+    find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" . \;
+    find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" . \;
 fi

rust-fil-sector-builder @ 1f1b7244

-Subproject commit 394b5b1bd87efbee6489a7d22dfb3b1dba697af0
+Subproject commit 1f1b7244fd2bb82cf4fe12106a1fc012f9d42998

binding/cgo_transforms.go → transforms.go

-package sectorbuilder
+package go_sectorbuilder
 
 import (
 	"unsafe"
 )
 
-// #cgo LDFLAGS: -L${SRCDIR}/../lib -lsector_builder_ffi
-// #cgo pkg-config: ${SRCDIR}/../lib/pkgconfig/sector_builder_ffi.pc
-// #include "../include/sector_builder_ffi.h"
+// #cgo LDFLAGS: ${SRCDIR}/libsector_builder_ffi.a
+// #cgo pkg-config: ${SRCDIR}/sector_builder_ffi.pc
+// #include "./sector_builder_ffi.h"
 import "C"
 
 // SingleProofPartitionProofLen denotes the number of bytes in a proof generated
@@ -81,9 +81,15 @@ func goPieceMetadata(src *C.sector_builder_ffi_FFIPieceMetadata, size C.size_t)
 
 	ptrs := (*[1 << 30]C.sector_builder_ffi_FFIPieceMetadata)(unsafe.Pointer(src))[:size:size]
 	for i := 0; i < int(size); i++ {
+		commPSlice := goBytes(&ptrs[i].comm_p[0], CommitmentBytesLen)
+		var commP [CommitmentBytesLen]byte
+		copy(commP[:], commPSlice)
+
 		ps[i] = PieceMetadata{
 			Key:            C.GoString(ptrs[i].piece_key),
 			Size:           uint64(ptrs[i].num_bytes),
+			CommP:          commP,
+			InclusionProof: goBytes(ptrs[i].piece_inclusion_proof_ptr, ptrs[i].piece_inclusion_proof_len),
 		}
 	}