Merge pull request #14 from filecoin-project/features/comprehensive-circle-ci

Features/comprehensive circle ci

.circleci/config.yml

 version: 2.1
 
 jobs:
-  build_linux:
+  build_and_test_linux:
     docker:
       - image: circleci/golang:1.12.1-stretch
-    working_directory: /go/src/github.com/filecoin-project/go-sectorbuilder
+    working_directory: ~/go/src/github.com/filecoin-project/go-sectorbuilder
     resource_class: xlarge
+    steps:
+      - configure_environment_variables
+      - run:
+          name: Install Rust toolchain
+          command: |
+            (sudo apt-get update && sudo apt-get install -y clang libssl-dev && which cargo && which rustc) || (curl https://sh.rustup.rs -sSf | sh -s -- -y)
+            rustc --version
+      - run:
+          name: Install jq
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y jq
+            jq --version
+      - checkout
+      - update_submodules
+      - build_project
+      - lint_project
+      - restore_parameter_cache
+      - fetch_filecoin_parameters
+      - save_parameter_cache
+      - test_project
+  build_and_test_darwin:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/go/src/github.com/filecoin-project/go-sectorbuilder
+    resource_class: medium
+    steps:
+      - configure_environment_variables
+      - run:
+          name: Install go
+          command: |
+            curl https://dl.google.com/go/go1.12.1.darwin-amd64.pkg -o /tmp/go.pkg && \
+            sudo installer -pkg /tmp/go.pkg -target /
+            go version
+      - run:
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
+      - run:
+          name: Install Rust toolchain
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+            rustc --version
+      - run:
+          name: Install jq
+          command: |
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install jq
+            jq --version
+      - checkout
+      - update_submodules
+      - build_project
+      - lint_project
+      - restore_parameter_cache
+      - fetch_filecoin_parameters
+      - save_parameter_cache
+      - test_project
+
+workflows:
+  version: 2
+  build_and_test_linux:
+    jobs:
+      - build_and_test_linux
+  build_and_test_darwin:
+    jobs:
+      - build_and_test_darwin
+
+commands:
+  configure_environment_variables:
     steps:
       - run:
           name: Configure environment variables
           command: |
+            echo 'export PATH="/usr/local/go/bin:${HOME}/.cargo/bin:${PATH}:${HOME}/go/bin:${HOME}/.bin"' >> $BASH_ENV
+            echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV
             echo 'export FILECOIN_PARAMETER_CACHE="${HOME}/filecoin-proof-parameters/"' >> $BASH_ENV
             echo 'export GO111MODULE=on' >> $BASH_ENV
-      - checkout
+  fetch_filecoin_parameters:
+    steps:
+      - run:
+          name: Fetch filecoin groth parameters
+          command: |
+            ./scripts/retry.sh 1 10 1000 ./scripts/with-dots.sh ./bin/paramfetch --params-for-sector-sizes=1024 --verbose
+  update_submodules:
+    steps:
       - run:
           name: Update submodules
           command: git submodule update --init --recursive
+  build_project:
+    steps:
       - run:
-          name: Build upstream project
+          name: Build project
           command: make
+  lint_project:
+    steps:
       - run:
-          name: Build project
-          command: go build .
-
-workflows:
-  version: 2
-  test_all:
-    jobs:
-      - build_linux
+          name: Lint project
+          command: go run github.com/golangci/golangci-lint/cmd/golangci-lint run
+  test_project:
+    steps:
+      - run:
+          name: Test project
+          command: go test
+  restore_parameter_cache:
+    steps:
+      - restore_cache:
+          keys:
+            - v1-proof-params-{{ checksum "./bin/paramfetch" }}
+  save_parameter_cache:
+    steps:
+      - save_cache:
+          key: v1-proof-params-{{ checksum "./bin/paramfetch" }}
+          paths:
+            - "~/filecoin-proof-parameters/"

.gitignore

@@ -2,3 +2,4 @@
 **/*.a
 **/*.pc
 .install-rust-fil-sector-builder
+bin/

bindings_test.go

@@ -52,6 +52,7 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	require.NoError(t, err)
 
 	stagedSectors, err := sb.GetAllStagedSectors(ptr)
+	require.NoError(t, err)
 	require.Equal(t, 1, len(stagedSectors))
 	stagedSector := stagedSectors[0]
 	require.Equal(t, uint64(1), stagedSector.SectorID)
@@ -78,9 +79,11 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 
 	// verify the PoSt
 	isValid, err = sb.VerifyPoSt(1024, [][32]byte{status.CommR}, [32]byte{}, proofs, faults)
+	require.NoError(t, err)
 	require.True(t, isValid)
 
 	sealedSectors, err := sb.GetAllSealedSectors(ptr)
+	require.NoError(t, err)
 	require.Equal(t, 1, len(sealedSectors), "expected to see one sealed sector")
 	sealedSector := sealedSectors[0]
 	require.Equal(t, uint64(1), sealedSector.SectorID)
@@ -92,6 +95,7 @@ func TestSectorBuilderLifecycle(t *testing.T) {
 	// unseal the sector and retrieve the client's piece, verifying that the
 	// retrieved bytes match what we originally wrote to the staged sector
 	unsealedPieceBytes, err := sb.ReadPieceFromSealedSector(ptr, "snoqualmie")
+	require.NoError(t, err)
 	require.Equal(t, pieceBytes, unsealedPieceBytes)
 }
 

go.mod

@@ -3,6 +3,7 @@ module github.com/filecoin-project/go-sectorbuilder
 go 1.12
 
 require (
+	github.com/golangci/golangci-lint v1.17.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/ipfs/go-log v0.0.1
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

install-rust-fil-sector-builder

@@ -16,6 +16,9 @@ if download_release_tarball tarball_path "${subm_dir}"; then
     cp "${tmp_dir}/include/sector_builder_ffi.h" .
     cp "${tmp_dir}/lib/libsector_builder_ffi.a" .
     cp "${tmp_dir}/lib/pkgconfig/sector_builder_ffi.pc" .
+
+    mkdir -p bin
+    cp "${tmp_dir}/bin/paramfetch" bin
 else
     echo "failed to find or obtain precompiled assets for ${subm_dir}, falling back to local build"
     build_from_source "${subm_dir}"
@@ -26,4 +29,13 @@ else
     find "${subm_dir}" -type f -name sector_builder_ffi.h -exec cp -- "{}" . \;
     find "${subm_dir}" -type f -name libsector_builder_ffi.a -exec cp -- "{}" . \;
     find "${subm_dir}" -type f -name sector_builder_ffi.pc -exec cp -- "{}" . \;
+
+    pushd $subm_dir
+
+    # TODO https://github.com/filecoin-project/rust-fil-sector-builder/issues/39
+    FP_VERSION=$(cat "Cargo.lock" | grep 'name = "filecoin-proofs"' -A1 | tail -n1 | cut -d' ' -f3 | tr -d '"')
+
+    cargo install --version $FP_VERSION filecoin-proofs --root '..' --bin paramfetch
+
+    popd
 fi

rust-fil-sector-builder @ 604b4876

-Subproject commit d92eeafbd7904ebf596b2b6b2b6a2e577f92f567
+Subproject commit 604b487673fa52744218abbe45972c1ee0b8d8cb

scripts/retry.sh

+#!/usr/bin/env bash
+
+# Inspired by https://gist.github.com/reacocard/28611bfaa2395072119464521d48729a
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Retry a command on a particular exit code, up to a max number of attempts,
+# with exponential backoff.
+# Invocation:
+#   err_retry exit_code attempts sleep_multiplier <command>
+# exit_code: The exit code to retry on.
+# attempts: The number of attempts to make.
+# sleep_millis: Multiplier for sleep between attempts. Examples:
+#     If multiplier is 1000, sleep intervals are 1, 4, 9, 16, etc. seconds.
+#     If multiplier is 5000, sleep intervals are 5, 20, 45, 80, 125, etc. seconds.
+
+exit_code=$1
+attempts=$2
+sleep_millis=$3
+shift 3
+
+for attempt in `seq 1 $attempts`; do
+    # This weird construction lets us capture return codes under -o errexit
+    "$@" && rc=$? || rc=$?
+
+    if [[ ! $rc -eq $exit_code ]]; then
+        exit $rc
+    fi
+
+    if [[ $attempt -eq $attempts ]]; then
+        exit $rc
+    fi
+
+    sleep_ms="$(($attempt * $attempt * $sleep_millis))"
+
+    sleep_seconds=$(echo "scale=2; ${sleep_ms}/1000" | bc)
+
+    >&2 echo "sleeping ${sleep_seconds}s and then retrying ($((attempt + 1))/${attempts})"
+
+    sleep "${sleep_seconds}"
+done

scripts/with-dots.sh

+#!/usr/bin/env bash
+
+trap cleanup EXIT
+
+cleanup() {
+  kill $DOT_PID
+}
+
+(
+  sleep 1
+  while true; do
+    (printf "." >&2)
+    sleep 1
+  done
+) &
+DOT_PID=$!
+
+$@