feat: implement election-post (#50)

* switch to feat/election-post  for rust-fil-sector-buidler

* update sector-builder

* update api to match election post

* update from upstream

* update upstream

* update upstream

* use branch for param fetch

* update upstream

* feat(candidates): provided generated candidates to PoSt generator

* ci(cache): bump cache key + selectively generate params

* fix(candidates): use generated candidates in PoSt verification

* fix(timeout): bump go test timeout

.circleci/config.yml

@@ -84,7 +84,7 @@ commands:
     steps:
       - run:
           name: Obtain filecoin groth parameters
-          command: ./paramcache --test-only
+          command: ./paramcache --params-for-sector-sizes=1024
           no_output_timeout: 30m
   update_submodules:
     steps:
@@ -100,7 +100,7 @@ commands:
           name: Ensure paramcache is installed to project root
           command: |
             test -f ./paramcache \
-              || (rustup run --install nightly cargo install filecoin-proofs --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch=master --bin=paramcache --root=./ \
+              || (rustup run --install nightly cargo install filecoin-proofs --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch="feat/election-post" --bin=paramcache --root=./ \
                 && mv ./bin/paramcache ./paramcache)
   lint_project:
     steps:
@@ -111,15 +111,16 @@ commands:
     steps:
       - run:
           name: Test project
-          command: RUST_LOG=info go test -p 1
+          command: RUST_LOG=info go test -p 1 -timeout 60m
+          no_output_timeout: 60m
   restore_parameter_cache:
     steps:
       - restore_cache:
           keys:
-            - v15-proof-params-{{ arch }}
+            - v17-proof-params-{{ arch }}
   save_parameter_cache:
     steps:
       - save_cache:
-          key: v15-proof-params-{{ arch }}
+          key: v17-proof-params-{{ arch }}
           paths:
             - "~/filecoin-proof-parameters/"

bindings.go

@@ -48,6 +48,13 @@ type SealSeed struct {
 	TicketBytes [32]byte
 }
 
+type Candidate struct {
+	SectorID             uint64
+	PartialTicket        [32]byte
+	Ticket               [32]byte
+	SectorChallengeIndex uint64
+}
+
 // NewSortedSectorInfo returns a SortedSectorInfo
 func NewSortedSectorInfo(sectorInfo ...SectorInfo) SortedSectorInfo {
 	fn := func(i, j int) bool {
@@ -226,7 +233,8 @@ func VerifyPoSt(
 	sectorInfo SortedSectorInfo,
 	challengeSeed [32]byte,
 	proof []byte,
-	faults []uint64,
+	winners []Candidate,
+	proverID [32]byte,
 ) (bool, error) {
 	defer elapsed("VerifyPoSt")()
 
@@ -259,8 +267,11 @@ func VerifyPoSt(
 	sectorIdsPtr, sectorIdsSize := cUint64s(sortedSectorIds)
 	defer C.free(unsafe.Pointer(sectorIdsPtr))
 
-	faultsPtr, faultsSize := cUint64s(faults)
-	defer C.free(unsafe.Pointer(faultsPtr))
+	winnersPtr, winnersSize := cCandidates(winners)
+	defer C.free(unsafe.Pointer(winnersPtr))
+
+	proverIDCBytes := C.CBytes(proverID[:])
+	defer C.free(proverIDCBytes)
 
 	// a mutable pointer to a VerifyPoStResponse C-struct
 	resPtr := C.sector_builder_ffi_reexported_verify_post(
@@ -268,12 +279,13 @@ func VerifyPoSt(
 		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedCBytes),
 		sectorIdsPtr,
 		sectorIdsSize,
-		faultsPtr,
-		faultsSize,
 		(*C.uint8_t)(flattenedCommRsCBytes),
 		C.size_t(len(flattened)),
 		(*C.uint8_t)(proofCBytes),
 		C.size_t(len(proof)),
+		winnersPtr,
+		winnersSize,
+		(*[32]C.uint8_t)(proverIDCBytes),
 	)
 	defer C.sector_builder_ffi_reexported_destroy_verify_post_response(resPtr)
 
@@ -634,12 +646,78 @@ func GetSectorSealingStatusByID(sectorBuilderPtr unsafe.Pointer, sectorID uint64
 	}
 }
 
+// FinalizeTicket creates an actual ticket from a partial ticket.
+func FinalizeTicket(partialTicket [32]byte) ([32]byte, error) {
+	defer elapsed("FinalizeTicket")()
+
+	partialTicketPtr := unsafe.Pointer(&(partialTicket)[0])
+	resPtr := C.sector_builder_ffi_reexported_finalize_ticket(
+		(*[32]C.uint8_t)(partialTicketPtr),
+	)
+	defer C.sector_builder_ffi_reexported_destroy_finalize_ticket_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return [32]byte{}, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return goCommitment(&resPtr.ticket[0]), nil
+}
+
+// GenerateCandidates creates a list of election candidates.
+func GenerateCandidates(
+	sectorBuilderPtr unsafe.Pointer,
+	sectorInfo SortedSectorInfo,
+	challengeSeed [CommitmentBytesLen]byte,
+	faults []uint64,
+) ([]Candidate, error) {
+	defer elapsed("GenerateCandidates")()
+
+	// CommRs and sector ids must be provided to C.verify_post in the same order
+	// that they were provided to the C.generate_post
+	sortedCommRs := make([][CommitmentBytesLen]byte, len(sectorInfo.Values()))
+	for idx, v := range sectorInfo.Values() {
+		sortedCommRs[idx] = v.CommR
+	}
+
+	// flattening the byte slice makes it easier to copy into the C heap
+	flattened := make([]byte, CommitmentBytesLen*len(sortedCommRs))
+	for idx, commR := range sortedCommRs {
+		copy(flattened[(CommitmentBytesLen*idx):(CommitmentBytesLen*(1+idx))], commR[:])
+	}
+
+	// copy the Go byte slice into C memory
+	cflattened := C.CBytes(flattened)
+	defer C.free(cflattened)
+
+	challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0])
+
+	faultsPtr, faultsSize := cUint64s(faults)
+	defer C.free(unsafe.Pointer(faultsPtr))
+
+	// a mutable pointer to a GenerateCandidatesResponse C-struct
+	resPtr := C.sector_builder_ffi_generate_candidates(
+		(*C.sector_builder_ffi_SectorBuilder)(sectorBuilderPtr),
+		(*C.uint8_t)(cflattened),
+		C.size_t(len(flattened)),
+		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedPtr),
+		faultsPtr,
+		faultsSize,
+	)
+	defer C.sector_builder_ffi_destroy_generate_candidates_response(resPtr)
+
+	if resPtr.status_code != 0 {
+		return nil, errors.New(C.GoString(resPtr.error_msg))
+	}
+
+	return goCandidates(resPtr.candidates_ptr, resPtr.candidates_len)
+}
+
 // GeneratePoSt produces a proof-of-spacetime for the provided replica commitments.
 func GeneratePoSt(
 	sectorBuilderPtr unsafe.Pointer,
 	sectorInfo SortedSectorInfo,
 	challengeSeed [CommitmentBytesLen]byte,
-	faults []uint64,
+	winners []Candidate,
 ) ([]byte, error) {
 	defer elapsed("GeneratePoSt")()
 
@@ -662,8 +740,8 @@ func GeneratePoSt(
 
 	challengeSeedPtr := unsafe.Pointer(&(challengeSeed)[0])
 
-	faultsPtr, faultsSize := cUint64s(faults)
-	defer C.free(unsafe.Pointer(faultsPtr))
+	winnersPtr, winnersSize := cCandidates(winners)
+	defer C.free(unsafe.Pointer(winnersPtr))
 
 	// a mutable pointer to a GeneratePoStResponse C-struct
 	resPtr := C.sector_builder_ffi_generate_post(
@@ -671,8 +749,8 @@ func GeneratePoSt(
 		(*C.uint8_t)(cflattened),
 		C.size_t(len(flattened)),
 		(*[CommitmentBytesLen]C.uint8_t)(challengeSeedPtr),
-		faultsPtr,
-		faultsSize,
+		winnersPtr,
+		winnersSize,
 	)
 	defer C.sector_builder_ffi_destroy_generate_post_response(resPtr)
 
@@ -680,7 +758,7 @@ func GeneratePoSt(
 		return nil, errors.New(C.GoString(resPtr.error_msg))
 	}
 
-	return goBytes(resPtr.proof_ptr, resPtr.proof_len), nil
+	return goBytes(resPtr.flattened_proofs_ptr, resPtr.flattened_proofs_len), nil
 }
 
 // AcquireSectorId returns a sector ID which can be used by out-of-band sealing.

bindings_test.go

@@ -178,12 +178,15 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 		CommR:    statusA.CommR,
 	})
 
+	candidates, err := sb.GenerateCandidates(ptr, sectorInfo, [32]byte{}, []uint64{})
+	require.NoError(t, err)
+
 	// generate a PoSt
-	proofs, err := sb.GeneratePoSt(ptr, sectorInfo, [32]byte{}, []uint64{})
+	proofs, err := sb.GeneratePoSt(ptr, sectorInfo, [32]byte{}, candidates)
 	require.NoError(t, err)
 
 	// verify the PoSt
-	isValid, err = sb.VerifyPoSt(1024, sectorInfo, [32]byte{}, proofs, []uint64{})
+	isValid, err = sb.VerifyPoSt(1024, sectorInfo, [32]byte{}, proofs, candidates, proverID)
 	require.NoError(t, err)
 	require.True(t, isValid)
 

rust-fil-sector-builder @ aa644623

-Subproject commit c0f5a817c63bd228c36fde904bf2ad3965e8a3d5
+Subproject commit aa6446237961eca1f6ca773a7d7debb39252d290

transforms.go

@@ -86,6 +86,26 @@ func cSealPreCommitOutput(src RawSealPreCommitOutput) C.sector_builder_ffi_FFISe
 	}
 }
 
+func cCandidates(src []Candidate) (*C.sector_builder_ffi_FFICandidate, C.size_t) {
+	srcCSizeT := C.size_t(len(src))
+
+	// allocate array in C heap
+	cCandidates := C.malloc(srcCSizeT * C.sizeof_sector_builder_ffi_FFICandidate)
+
+	// create a Go slice backed by the C-array
+	pp := (*[1 << 30]C.sector_builder_ffi_FFICandidate)(cCandidates)
+	for i, v := range src {
+		pp[i] = C.sector_builder_ffi_FFICandidate{
+			sector_id:              C.uint64_t(v.SectorID),
+			partial_ticket:         *(*[32]C.uint8_t)(unsafe.Pointer(&v.PartialTicket)),
+			ticket:                 *(*[32]C.uint8_t)(unsafe.Pointer(&v.Ticket)),
+			sector_challenge_index: C.uint64_t(v.SectorChallengeIndex),
+		}
+	}
+
+	return (*C.sector_builder_ffi_FFICandidate)(cCandidates), srcCSizeT
+}
+
 func goBytes(src *C.uint8_t, size C.size_t) []byte {
 	return C.GoBytes(unsafe.Pointer(src), C.int(size))
 }
@@ -97,6 +117,29 @@ func goSealTicket(src C.sector_builder_ffi_FFISealTicket) SealTicket {
 	}
 }
 
+func goCandidates(src *C.sector_builder_ffi_FFICandidate, size C.size_t) ([]Candidate, error) {
+	candidates := make([]Candidate, size)
+	if src == nil || size == 0 {
+		return candidates, nil
+	}
+
+	ptrs := (*[1 << 30]C.sector_builder_ffi_FFICandidate)(unsafe.Pointer(src))[:size:size]
+	for i := 0; i < int(size); i++ {
+		candidates[i] = goCandidate(ptrs[i])
+	}
+
+	return candidates, nil
+}
+
+func goCandidate(src C.sector_builder_ffi_FFICandidate) Candidate {
+	return Candidate{
+		SectorID:             uint64(src.sector_id),
+		PartialTicket:        goCommitment(&src.partial_ticket[0]),
+		Ticket:               goCommitment(&src.ticket[0]),
+		SectorChallengeIndex: uint64(src.sector_challenge_index),
+	}
+}
+
 func goRawSealPreCommitOutput(src C.sector_builder_ffi_FFISealPreCommitOutput) RawSealPreCommitOutput {
 	return RawSealPreCommitOutput{
 		CommD:     goCommitment(&src.comm_d[0]),