Merge pull request #16 from filecoin-project/feat/rational-post

updates to consume new Rational PoSt API

.circleci/config.yml

@@ -106,7 +106,7 @@ commands:
     steps:
       - run:
           name: Test project
-          command: go test
+          command: RUST_LOG=info go test
   restore_parameter_cache:
     steps:
       - restore_cache:

bindings.go

 package go_sectorbuilder
 
 import (
+	"bytes"
+	"sort"
 	"time"
 	"unsafe"
 
@@ -22,6 +24,35 @@ func elapsed(what string) func() {
 	}
 }
 
+// SortedSectorInfo is a slice of SectorInfo sorted (lexicographically,
+// ascending) by replica commitment (CommR).
+type SortedSectorInfo struct {
+	f []SectorInfo
+}
+
+// NewSortedSectorInfo returns a SortedSectorInfo
+func NewSortedSectorInfo(sectorInfo ...SectorInfo) SortedSectorInfo {
+	fn := func(i, j int) bool {
+		return bytes.Compare(sectorInfo[i].CommR[:], sectorInfo[j].CommR[:]) == -1
+	}
+
+	sort.Slice(sectorInfo[:], fn)
+
+	return SortedSectorInfo{
+		f: sectorInfo,
+	}
+}
+
+// Values returns the sorted SectorInfo as a slice
+func (s *SortedSectorInfo) Values() []SectorInfo {
+	return s.f
+}
+
+type SectorInfo struct {
+	SectorID uint64
+	CommR [CommitmentBytesLen]byte
+}
+
 // CommitmentBytesLen is the number of bytes in a CommR, CommD, CommP, and CommRStar.
 const CommitmentBytesLen = 32
 
@@ -71,7 +102,7 @@ func VerifySeal(
 	commD [CommitmentBytesLen]byte,
 	commRStar [CommitmentBytesLen]byte,
 	proverID [31]byte,
-	sectorID [31]byte,
+	sectorID uint64,
 	proof []byte,
 ) (bool, error) {
 	defer elapsed("VerifySeal")()
@@ -91,9 +122,6 @@ func VerifySeal(
 	proverIDCBytes := C.CBytes(proverID[:])
 	defer C.free(proverIDCBytes)
 
-	sectorIDCbytes := C.CBytes(sectorID[:])
-	defer C.free(sectorIDCbytes)
-
 	// a mutable pointer to a VerifySealResponse C-struct
 	resPtr := C.sector_builder_ffi_verify_seal(
 		C.uint64_t(sectorSize),
@@ -101,7 +129,7 @@ func VerifySeal(
 		(*[CommitmentBytesLen]C.uint8_t)(commDCBytes),
 		(*[CommitmentBytesLen]C.uint8_t)(commRStarCBytes),
 		(*[31]C.uint8_t)(proverIDCBytes),
-		(*[31]C.uint8_t)(sectorIDCbytes),
+		C.uint64_t(sectorID),
 		(*C.uint8_t)(proofCBytes),
 		C.size_t(len(proof)),
 	)
@@ -118,25 +146,25 @@ func VerifySeal(
 // inputs were derived was valid, and false if not.
 func VerifyPoSt(
 	sectorSize uint64,
-	sortedCommRs [][CommitmentBytesLen]byte,
-	challengeSeed [CommitmentBytesLen]byte,
-	proofs [][]byte,
+	sectorInfo SortedSectorInfo,
+	challengeSeed [32]byte,
+	proof []byte,
 	faults []uint64,
 ) (bool, error) {
 	defer elapsed("VerifyPoSt")()
 
-	// validate verification request
-	if len(proofs) == 0 {
-		return false, errors.New("must provide at least one proof to verify")
+	// CommRs and sector ids must be provided to C.verify_post in the same order
+	// that they were provided to the C.generate_post
+	sortedCommRs := make([][CommitmentBytesLen]byte, len(sectorInfo.Values()))
+	sortedSectorIds := make([]uint64, len(sectorInfo.Values()))
+	for idx, v := range sectorInfo.Values() {
+		sortedCommRs[idx] = v.CommR
+		sortedSectorIds[idx] = v.SectorID
 	}
 
-	// CommRs must be provided to C.verify_post in the same order that they were
-	// provided to the C.generate_post
-	commRs := sortedCommRs
-
 	// flattening the byte slice makes it easier to copy into the C heap
-	flattened := make([]byte, CommitmentBytesLen*len(commRs))
-	for idx, commR := range commRs {
+	flattened := make([]byte, CommitmentBytesLen*len(sortedCommRs))
+	for idx, commR := range sortedCommRs {
 		copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
 	}
 
@@ -147,24 +175,28 @@ func VerifyPoSt(
 	challengeSeedCBytes := C.CBytes(challengeSeed[:])
 	defer C.free(challengeSeedCBytes)
 
-	proofPartitions, proofsPtr, proofsLen := cPoStProofs(proofs)
-	defer C.free(unsafe.Pointer(proofsPtr))
+	proofCBytes := C.CBytes(proof)
+	defer C.free(proofCBytes)
 
 	// allocate fixed-length array of uint64s in C heap
+	sectorIdsPtr, sectorIdsSize := cUint64s(sortedSectorIds)
+	defer C.free(unsafe.Pointer(sectorIdsPtr))
+
 	faultsPtr, faultsSize := cUint64s(faults)
 	defer C.free(unsafe.Pointer(faultsPtr))
 
 	// a mutable pointer to a VerifyPoStResponse C-struct
 	resPtr := C.sector_builder_ffi_verify_post(
 		C.uint64_t(sectorSize),
-		proofPartitions,
-		(*C.uint8_t)(flattenedCommRsCBytes),
-		C.size_t(len(flattened)),
 		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
-		proofsPtr,
-		proofsLen,
+		sectorIdsPtr,
+		sectorIdsSize,
 		faultsPtr,
 		faultsSize,
+		(*C.uint8_t)(flattenedCommRsCBytes),
+		C.size_t(len(flattened)),
+		(*C.uint8_t)(proofCBytes),
+		C.size_t(len(proof)),
 	)
 	defer C.sector_builder_ffi_destroy_verify_post_response(resPtr)
 
@@ -210,7 +242,7 @@ func InitSectorBuilder(
 	cSealedSectorDir := C.CString(sealedSectorDir)
 	defer C.free(unsafe.Pointer(cSealedSectorDir))
 
-	class, err := cSectorClass(sectorSize, poRepProofPartitions, poStProofPartitions)
+	class, err := cSectorClass(sectorSize, poRepProofPartitions)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get sector class")
 	}
@@ -418,15 +450,22 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
 // GeneratePoSt produces a proof-of-spacetime for the provided replica commitments.
 func GeneratePoSt(
 	sectorBuilderPtr unsafe.Pointer,
-	sortedCommRs [][CommitmentBytesLen]byte,
+	sectorInfo SortedSectorInfo,
 	challengeSeed [CommitmentBytesLen]byte,
-) ([][]byte, []uint64, error) {
+	faults []uint64,
+) ([]byte, error) {
 	defer elapsed("GeneratePoSt")()
 
+	// CommRs and sector ids must be provided to C.verify_post in the same order
+	// that they were provided to the C.generate_post
+	sortedCommRs := make([][CommitmentBytesLen]byte, len(sectorInfo.Values()))
+	for idx, v := range sectorInfo.Values() {
+		sortedCommRs[idx] = v.CommR
+	}
+
 	// flattening the byte slice makes it easier to copy into the C heap
-	commRs := sortedCommRs
-	flattened := make([]byte, CommitmentBytesLen*len(commRs))
-	for idx, commR := range commRs {
+	flattened := make([]byte, CommitmentBytesLen*len(sortedCommRs))
+	for idx, commR := range sortedCommRs {
 		copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
 	}
 
@@ -436,25 +475,25 @@ func GeneratePoSt(
 
 	challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0])
 
+	faultsPtr, faultsSize := cUint64s(faults)
+	defer C.free(unsafe.Pointer(faultsPtr))
+
 	// a mutable pointer to a GeneratePoStResponse C-struct
 	resPtr := C.sector_builder_ffi_generate_post(
 		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
 		(*C.uint8_t)(cflattened),
 		C.size_t(len(flattened)),
 		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedPtr),
+		faultsPtr,
+		faultsSize,
 	)
 	defer C.sector_builder_ffi_destroy_generate_post_response(resPtr)
 
 	if resPtr.status_code != 0 {
-		return nil, nil, errors.New(C.GoString(resPtr.error_msg))
-	}
-
-	proofs, err := goPoStProofs(resPtr.proof_partitions, resPtr.flattened_proofs_ptr, resPtr.flattened_proofs_len)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "failed to convert to []PoStProof")
+		return nil, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	return proofs, goUint64s(resPtr.faults_ptr, resPtr.faults_len), nil
+	return goBytes(resPtr.proof_ptr, resPtr.proof_len), nil
 }
 
 // VerifyPieceInclusionProof returns true if the piece inclusion proof is valid

bindings_test.go

@@ -3,7 +3,6 @@ package go_sectorbuilder_test
 import (
 	"bytes"
 	"crypto/rand"
-	"encoding/binary"
 	"errors"
 	"io"
 	"io/ioutil"
@@ -64,7 +63,7 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	require.Equal(t, 1, len(status.Pieces), "expected to see the one piece we added")
 
 	// verify the seal proof
-	isValid, err := sb.VerifySeal(1024, status.CommR, status.CommD, status.CommRStar, [31]byte{}, sectorIdAsBytes(sectorID), status.Proof)
+	isValid, err := sb.VerifySeal(1024, status.CommR, status.CommD, status.CommRStar, [31]byte{}, sectorID, status.Proof)
 	require.NoError(t, err)
 	require.True(t, isValid)
 
@@ -73,12 +72,18 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	require.NoError(t, err)
 	require.True(t, isValid)
 
+	// enforces sort ordering of SectorInfo tuples
+	sectorInfo := sb.NewSortedSectorInfo(sb.SectorInfo{
+		SectorID: status.SectorID,
+		CommR:    status.CommR,
+	})
+
 	// generate a PoSt
-	proofs, faults, err := sb.GeneratePoSt(ptr, [][32]byte{status.CommR}, [32]byte{})
+	proofs, err := sb.GeneratePoSt(ptr, sectorInfo, [32]byte{}, []uint64{})
 	require.NoError(t, err)
 
 	// verify the PoSt
-	isValid, err = sb.VerifyPoSt(1024, [][32]byte{status.CommR}, [32]byte{}, proofs, faults)
+	isValid, err = sb.VerifyPoSt(1024, sectorInfo, [32]byte{}, proofs, []uint64{})
 	require.NoError(t, err)
 	require.True(t, isValid)
 
@@ -124,16 +129,6 @@ func pollForSectorSealingStatus(ptr unsafe.Pointer, sectorID uint64, sealStatusC
 	}
 }
 
-func sectorIdAsBytes(sectorID uint64) [31]byte {
-	slice := make([]byte, 31)
-	binary.LittleEndian.PutUint64(slice, sectorID)
-
-	var sectorIDAsBytes [31]byte
-	copy(sectorIDAsBytes[:], slice)
-
-	return sectorIDAsBytes
-}
-
 func requireTempFilePath(t *testing.T, fileContentsReader io.Reader) string {
 	file, err := ioutil.TempFile("", "")
 	require.NoError(t, err)

install-rust-fil-sector-builder

@@ -20,7 +20,7 @@ if download_release_tarball tarball_path "${subm_dir}"; then
     mkdir -p bin
     cp "${tmp_dir}/bin/paramfetch" bin
 else
-    echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build"
+    (>&2 echo "failed to find or obtain precompiled assets for ${subm_dir} - falling back to local build")
     build_from_source "${subm_dir}"
 
     mkdir -p include
@@ -32,10 +32,19 @@ else
 
     pushd $subm_dir
 
-    # TODO https://github.com/filecoin-project/rust-fil-sector-builder/issues/39
-    FP_VERSION=$(cat "Cargo.lock" | grep 'name = "filecoin-proofs"' -A1 | tail -n1 | cut -d' ' -f3 | tr -d '"')
-
-    cargo install --version $FP_VERSION filecoin-proofs --root '..' --bin paramfetch
+    s="WARNING: paramfetch cannot be built from rust-fil-sector-builder "
+    s+="sources. Installing from HEAD of rust-fil-proofs master. The installed "
+    s+="version of paramfetch may be incompatible with your version of "
+    s+="rust-fil-sector-builder, which could result in you downloading "
+    s+="incompatible Groth parameters and verifying keys."
+
+    (>&2 echo s)
+    cargo install filecoin-proofs \
+      --bin paramfetch \
+      --force \
+      --git=https://github.com/filecoin-project/rust-fil-proofs.git \
+      --branch=master \
+      --root ".."
 
     popd
 fi

rust-fil-sector-builder @ 0282a636

-Subproject commit 604b487673fa52744218abbe45972c1ee0b8d8cb
+Subproject commit 0282a636f7911bcfd0348af5494da5932ac5808b

scripts/retry.sh

@@ -37,7 +37,7 @@ for attempt in `seq 1 $attempts`; do
 
     sleep_seconds=$(echo "scale=2; ${sleep_ms}/1000" | bc)
 
-    >&2 echo "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})"
+    (>&2 echo "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})")
 
     sleep "${sleep_seconds}"
 done

transforms.go

@@ -16,22 +16,6 @@ import "C"
 // with the number of partitions used when creating that proof.
 const SingleProofPartitionProofLen = 192
 
-func cPoStProofs(src [][]byte) (C.uint8_t, *C.uint8_t, C.size_t) {
-	proofSize := len(src[0])
-
-	flattenedLen := C.size_t(proofSize * len(src))
-
-	// flattening the byte slice makes it easier to copy into the C heap
-	flattened := make([]byte, flattenedLen)
-	for idx, proof := range src {
-		copy(flattened[(proofSize*idx):(proofSize*(1+idx))], proof[:])
-	}
-
-	proofPartitions := proofSize / SingleProofPartitionProofLen
-
-	return C.uint8_t(proofPartitions), (*C.uint8_t)(C.CBytes(flattened)), flattenedLen
-}
-
 func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
 	srcCSizeT := C.size_t(len(src))
 
@@ -47,11 +31,10 @@ func cUint64s(src []uint64) (*C.uint64_t, C.size_t) {
 	return (*C.uint64_t)(cUint64s), srcCSizeT
 }
 
-func cSectorClass(sectorSize uint64, poRepProofPartitions uint8, poStProofPartitions uint8) (C.sector_builder_ffi_FFISectorClass, error) {
+func cSectorClass(sectorSize uint64, poRepProofPartitions uint8) (C.sector_builder_ffi_FFISectorClass, error) {
 	return C.sector_builder_ffi_FFISectorClass{
 		sector_size:            C.uint64_t(sectorSize),
 		porep_proof_partitions: C.uint8_t(poRepProofPartitions),
-		post_proof_partitions:  C.uint8_t(poStProofPartitions),
 	}, nil
 }
 
@@ -137,25 +120,3 @@ func goPieceMetadata(src *C.sector_builder_ffi_FFIPieceMetadata, size C.size_t)
 
 	return ps, nil
 }
-
-func goPoStProofs(partitions C.uint8_t, src *C.uint8_t, size C.size_t) ([][]byte, error) {
-	tmp := goBytes(src, size)
-
-	arraySize := len(tmp)
-	chunkSize := int(partitions) * SingleProofPartitionProofLen
-
-	out := make([][]byte, arraySize/chunkSize)
-	for i := 0; i < len(out); i++ {
-		out[i] = append([]byte{}, tmp[i*chunkSize:(i+1)*chunkSize]...)
-	}
-
-	return out, nil
-}
-
-func goUint64s(src *C.uint64_t, size C.size_t) []uint64 {
-	out := make([]uint64, size)
-	if src != nil {
-		copy(out, (*(*[1 << 30]uint64)(unsafe.Pointer(src)))[:size:size])
-	}
-	return out
-}