Merge pull request #20 from filecoin-project/feat/47-paramcache-not-paramfetch-rfsb

use paramcache instead of paramfetch

.circleci/config.yml

@@ -23,8 +23,9 @@ jobs:
       - update_submodules
       - build_project
       - lint_project
+      - generate_parameter_checksum
       - restore_parameter_cache
-      - fetch_filecoin_parameters
+      - obtain_filecoin_parameters
       - save_parameter_cache
       - test_project
   build_and_test_darwin:
@@ -41,8 +42,8 @@ jobs:
             sudo installer -pkg /tmp/go.pkg -target /
             go version
       - run:
-          name: Install pkg-config
-          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
+          name: Install pkg-config and md5sum
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum
       - run:
           name: Install Rust toolchain
           command: |
@@ -57,18 +58,17 @@ jobs:
       - update_submodules
       - build_project
       - lint_project
+      - generate_parameter_checksum
       - restore_parameter_cache
-      - fetch_filecoin_parameters
+      - obtain_filecoin_parameters
       - save_parameter_cache
       - test_project
 
 workflows:
   version: 2
-  build_and_test_linux:
+  test_all:
     jobs:
       - build_and_test_linux
-  build_and_test_darwin:
-    jobs:
       - build_and_test_darwin
 
 commands:
@@ -81,12 +81,13 @@ commands:
             echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
             echo 'export FILECOIN_PARAMETER_CACHE="${HOME}/filecoin-proof-parameters/"' >> $BASH_ENV
             echo 'export GO111MODULE=on' >> $BASH_ENV
-  fetch_filecoin_parameters:
+            echo 'export RUST_LOG=info' >> $BASH_ENV
+  obtain_filecoin_parameters:
     steps:
       - run:
-          name: Fetch filecoin groth parameters
-          command: |
-            ./scripts/retry.sh 1 10 1000 ./scripts/with-dots.sh ./bin/paramfetch --params-for-sector-sizes=1024 --verbose
+          name: Obtain filecoin groth parameters
+          command: ./paramcache --test-only
+          no_output_timeout: 30m
   update_submodules:
     steps:
       - run:
@@ -97,6 +98,12 @@ commands:
       - run:
           name: Build project
           command: make
+      - run:
+          name: Ensure paramcache is installed to project root
+          command: |
+            test -f ./paramcache \
+              || (rustup run --install nightly cargo install filecoin-proofs --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch=master --bin=paramcache --root=./ \
+                && mv ./bin/paramcache ./paramcache)
   lint_project:
     steps:
       - run:
@@ -111,10 +118,16 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - v1-proof-params-{{ checksum "./bin/paramfetch" }}
+            - v1-proof-params-{{ checksum "parameter-checksum.txt" }}
   save_parameter_cache:
     steps:
       - save_cache:
-          key: v1-proof-params-{{ checksum "./bin/paramfetch" }}
+          key: v1-proof-params-{{ checksum "parameter-checksum.txt" }}
           paths:
             - "~/filecoin-proof-parameters/"
+  generate_parameter_checksum:
+    steps:
+      - run:
+          name: create parameter cache checksum
+          command: md5sum ./paramcache | cut -d ' ' -f 1 >> parameter-checksum.txt
+

.gitignore

@@ -3,3 +3,5 @@
 **/*.pc
 .install-rust-fil-sector-builder
 bin/
+.crates.toml
+**/paramcache

install-rust-fil-sector-builder

@@ -17,8 +17,7 @@ if download_release_tarball tarball_path "${subm_dir}"; then
     cp "${tmp_dir}/lib/libsector_builder_ffi.a" .
     cp "${tmp_dir}/lib/pkgconfig/sector_builder_ffi.pc" .
 
-    mkdir -p bin
-    cp "${tmp_dir}/bin/paramfetch" bin
+    cp "${tmp_dir}/bin/paramcache" .
 else
     (>&2 echo "failed to find or obtain precompiled assets for ${subm_dir} - falling back to local build")
     build_from_source "${subm_dir}"
@@ -30,21 +29,5 @@ else
     find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" . \;
     find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" . \;
 
-    pushd $subm_dir
-
-    s="WARNING: paramfetch cannot be built from rust-fil-sector-builder "
-    s+="sources. Installing from HEAD of rust-fil-proofs master. The installed "
-    s+="version of paramfetch may be incompatible with your version of "
-    s+="rust-fil-sector-builder, which could result in you downloading "
-    s+="incompatible Groth parameters and verifying keys."
-
-    (>&2 echo s)
-    cargo install filecoin-proofs \
-      --bin paramfetch \
-      --force \
-      --git=https://github.com/filecoin-project/rust-fil-proofs.git \
-      --branch=master \
-      --root ".."
-
-    popd
+    (>&2 echo "WARNING: paramcache was not installed - you may wish to 'cargo install' it")
 fi

rust-fil-sector-builder @ 51c64008

-Subproject commit 60ff4b59300ee2e49eadacedd15979e8ec78613b
+Subproject commit 51c64008779b28e0d9b39f0a559a536263fd7793

scripts/retry.sh

-#!/usr/bin/env bash
-
-# Inspired by https://gist.github.com/reacocard/28611bfaa2395072119464521d48729a
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Retry a command on a particular exit code, up to a max number of attempts,
-# with exponential backoff.
-# Invocation:
-#   err_retry exit_code attempts sleep_multiplier <command>
-# exit_code: The exit code to retry on.
-# attempts: The number of attempts to make.
-# sleep_millis: Multiplier for sleep between attempts. Examples:
-#     If multiplier is 1000, sleep intervals are 1, 4, 9, 16, etc. seconds.
-#     If multiplier is 5000, sleep intervals are 5, 20, 45, 80, 125, etc. seconds.
-
-exit_code=$1
-attempts=$2
-sleep_millis=$3
-shift 3
-
-for attempt in `seq 1 $attempts`; do
-    # This weird construction lets us capture return codes under -o errexit
-    "$@" && rc=$? || rc=$?
-
-    if [[ ! $rc -eq $exit_code ]]; then
-        exit $rc
-    fi
-
-    if [[ $attempt -eq $attempts ]]; then
-        exit $rc
-    fi
-
-    sleep_ms="$(($attempt * $attempt * $sleep_millis))"
-
-    sleep_seconds=$(echo "scale=2; ${sleep_ms}/1000" | bc)
-
-    (>&2 echo "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})")
-
-    sleep "${sleep_seconds}"
-done

scripts/with-dots.sh

-#!/usr/bin/env bash
-
-trap cleanup EXIT
-
-cleanup() {
-  kill $DOT_PID
-}
-
-(
-  sleep 1
-  while true; do
-    (printf "." >&2)
-    sleep 1
-  done
-) &
-DOT_PID=$!
-
-$@