consume extracted paramfetch + basic CircleCI config (#63)

* consume newly-extracted go-paramfetch

* add basic CircleCI config, and use TestDownloadParams to hydrate cache before running tests

* make deps, not make deps lotus

* use Makefile target when testing + set env variable for verbose rust logging

* drop gosumtest

* drop short mode

* add some documentation

.circleci/config.yml

+version: 2.1
+orbs:
+  go: gotest/tools@0.0.9
+
+executors:
+  golang:
+    docker:
+      - image: circleci/golang:1.13
+    resource_class: 2xlarge
+
+commands:
+  install-deps:
+    steps:
+      - go/install-ssh
+      - go/install: {package: git}
+  prepare:
+    parameters:
+      linux:
+        default: true
+        description: is a linux build environment?
+        type: boolean
+      darwin:
+        default: false
+        description: is a darwin build environment?
+        type: boolean
+    steps:
+      - checkout
+      - when:
+          condition: << parameters.linux >>
+          steps:
+            - run: sudo apt-get update
+            - run: sudo apt-get install ocl-icd-opencl-dev
+      - run: git submodule sync
+      - run: git submodule update --init
+  download-params:
+    steps:
+      - run:
+          command: |
+            cat go.mod | grep go-paramfetch > go-paramfetch-checksum.txt
+      - restore_cache:
+          name: Restore parameters cache
+          keys:
+            - 'groth-params-and-keys-{{ checksum "go-paramfetch-checksum.txt" }}'
+          paths:
+            - /var/tmp/filecoin-proof-parameters/
+      - run:
+          name: Download Groth parameters and keys for 1KiB sector size (hard-coded in test)
+          command: make fetch-params
+      - save_cache:
+          name: Save parameters cache
+          key: 'groth-params-and-keys-{{ checksum "go-paramfetch-checksum.txt" }}'
+          paths:
+            - /var/tmp/filecoin-proof-parameters/
+
+jobs:
+  mod-tidy-check:
+    executor: golang
+    steps:
+      - install-deps
+      - prepare
+      - go/mod-download
+      - go/mod-tidy-check
+
+  build-all:
+    executor: golang
+    steps:
+      - install-deps
+      - prepare
+      - go/mod-download
+      - restore_cache:
+          name: restore go mod cache
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
+      - run:
+          command: make build
+
+  test: &test
+    description: |
+      Run tests.
+    parameters:
+      executor:
+        type: executor
+        default: golang
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - go/mod-download
+      - restore_cache:
+          name: restore go mod cache
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
+      - run:
+          command: make deps
+          no_output_timeout: 30m
+      - download-params
+      - run:
+          name: run tests
+          command: make test
+          no_output_timeout: 60m
+      - save_cache:
+          name: save go mod cache
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
+          paths:
+            - "~/go/pkg"
+            - "~/go/src/github.com"
+            - "~/go/src/golang.org"
+
+  build-macos:
+    description: build with Darwin
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/go/src/github.com/filecoin-project/go-sectorbuilder
+    steps:
+      - prepare:
+          linux: false
+          darwin: true
+      - run:
+          name: Install go
+          command: |
+            curl -O https://dl.google.com/go/go1.13.4.darwin-amd64.pkg && \
+            sudo installer -pkg go1.13.4.darwin-amd64.pkg -target /
+      - run:
+          name: Install pkg-config
+          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config
+      - run: go version
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run:
+          name: Install jq
+          command: |
+            mkdir $HOME/.bin
+            curl --location https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 --output $HOME/.bin/jq
+            chmod +x $HOME/.bin/jq
+      - restore_cache:
+          name: restore go mod and cargo cache
+          key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/go-sectorbuilder/go.sum" }}
+      - install-deps
+      - go/mod-download
+      - run:
+          command: make build
+          no_output_timeout: 30m
+      - save_cache:
+          name: save cargo cache
+          key: v1-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/go-sectorbuilder/go.sum" }}
+          paths:
+            - "~/.rustup"
+            - "~/.cargo"
+
+  lint: &lint
+    description: |
+      Run golangci-lint.
+    parameters:
+      executor:
+        type: executor
+        default: golang
+      golangci-lint-version:
+        type: string
+        default: 1.17.1
+      concurrency:
+        type: string
+        default: '2'
+        description: |
+          Concurrency used to run linters. Defaults to 2 because NumCPU is not
+          aware of container CPU limits.
+      args:
+        type: string
+        default: ''
+        description: |
+          Arguments to pass to golangci-lint
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - go/mod-download
+      - run:
+          command: make deps
+          no_output_timeout: 30m
+      - go/install-golangci-lint:
+          gobin: $HOME/.local/bin
+          version: << parameters.golangci-lint-version >>
+      - run:
+          name: Lint
+          command: |
+            $HOME/.local/bin/golangci-lint run -v \
+              --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
+  lint-all:
+    <<: *lint
+
+workflows:
+  version: 2.1
+  ci:
+    jobs:
+      - lint-changes:
+          args: "--new-from-rev origin/master"
+      - test
+      - mod-tidy-check
+      - build-all
+      - build-macos

Makefile

@@ -38,17 +38,21 @@ deps: $(BUILD_DEPS)
 .PHONY: deps
 
 test: $(BUILD_DEPS)
-	go test -v $(GOFLAGS) ./...
+	RUST_LOG=info go test -v -timeout 120m ./...
 .PHONY: test
 
 lint: $(BUILD_DEPS)
-	golangci-lint run -v  --concurrency 2 --new-from-rev origin/master
+	golangci-lint run -v --concurrency 2 --new-from-rev origin/master
 .PHONY: lint
 
 build: $(BUILD_DEPS)
 	go build -v $(GOFLAGS) ./...
 .PHONY: build
 
+fetch-params: $(BUILD_DEPS)
+	go test -run=^TestDownloadParams
+.PHONY: fetch-params
+
 clean:
 	rm -rf $(CLEAN)
 	-$(MAKE) -C $(FFI_PATH) clean

go.mod

@@ -3,10 +3,10 @@ module github.com/filecoin-project/go-sectorbuilder
 go 1.13
 
 require (
-	github.com/GeertJohan/go.rice v1.0.0
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590
 	github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f
+	github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/ipfs/go-cid v0.0.4 // indirect
@@ -16,8 +16,6 @@ require (
 	github.com/jbenet/goprocess v0.1.3 // indirect
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-isatty v0.0.9 // indirect
-	github.com/mattn/go-runewidth v0.0.4 // indirect
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
 	github.com/minio/sha256-simd v0.1.1 // indirect
 	github.com/mr-tron/base58 v1.1.3 // indirect
 	github.com/otiai10/copy v1.0.2
@@ -27,11 +25,9 @@ require (
 	github.com/stretchr/testify v1.4.0
 	github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 // indirect
 	go.opencensus.io v0.22.2
-	go.uber.org/multierr v1.4.0
 	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
 	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
-	gopkg.in/cheggaaa/pb.v1 v1.0.28
 )
 
 replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0

go.sum

@@ -17,6 +17,8 @@ github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2j
 github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
+github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
+github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -79,8 +81,8 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
-github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=

paramfetch/paramfetch.go

-package paramfetch
-
-import (
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-
-	rice "github.com/GeertJohan/go.rice"
-	logging "github.com/ipfs/go-log"
-	"github.com/minio/blake2b-simd"
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-	pb "gopkg.in/cheggaaa/pb.v1"
-)
-
-var log = logging.Logger("paramfetch")
-
-//const gateway = "http://198.211.99.118/ipfs/"
-const gateway = "https://ipfs.io/ipfs/"
-const paramdir = "/var/tmp/filecoin-proof-parameters"
-const dirEnv = "FIL_PROOFS_PARAMETER_CACHE"
-
-type paramFile struct {
-	Cid        string `json:"cid"`
-	Digest     string `json:"digest"`
-	SectorSize uint64 `json:"sector_size"`
-}
-
-type fetch struct {
-	wg      sync.WaitGroup
-	fetchLk sync.Mutex
-
-	errs []error
-}
-
-func getParamDir() string {
-	if os.Getenv(dirEnv) == "" {
-		return paramdir
-	}
-	return os.Getenv(dirEnv)
-}
-
-func GetParams(storageSize uint64) error {
-	if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-
-	var params map[string]paramFile
-
-	paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json")
-	if err := json.Unmarshal(paramBytes, &params); err != nil {
-		return err
-	}
-
-	ft := &fetch{}
-
-	for name, info := range params {
-		if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") {
-			continue
-		}
-
-		ft.maybeFetchAsync(name, info)
-	}
-
-	return ft.wait()
-}
-
-func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
-	ft.wg.Add(1)
-
-	go func() {
-		defer ft.wg.Done()
-
-		path := filepath.Join(getParamDir(), name)
-
-		err := ft.checkFile(path, info)
-		if !os.IsNotExist(err) && err != nil {
-			log.Warn(err)
-		}
-		if err == nil {
-			return
-		}
-
-		ft.fetchLk.Lock()
-		defer ft.fetchLk.Unlock()
-
-		if err := doFetch(path, info); err != nil {
-			ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err))
-			return
-		}
-		err = ft.checkFile(path, info)
-		if err != nil {
-			ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err))
-			err := os.Remove(path)
-			if err != nil {
-				ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err))
-			}
-		}
-	}()
-}
-
-func (ft *fetch) checkFile(path string, info paramFile) error {
-	if os.Getenv("TRUST_PARAMS") == "1" {
-		log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
-		return nil
-	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	h := blake2b.New512()
-	if _, err := io.Copy(h, f); err != nil {
-		return err
-	}
-
-	sum := h.Sum(nil)
-	strSum := hex.EncodeToString(sum[:16])
-	if strSum == info.Digest {
-		log.Infof("Parameter file %s is ok", path)
-		return nil
-	}
-
-	return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
-}
-
-func (ft *fetch) wait() error {
-	ft.wg.Wait()
-	return multierr.Combine(ft.errs...)
-}
-
-func doFetch(out string, info paramFile) error {
-	gw := os.Getenv("IPFS_GATEWAY")
-	if gw == "" {
-		gw = gateway
-	}
-	log.Infof("Fetching %s from %s", out, gw)
-
-	outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-	if err != nil {
-		return err
-	}
-	defer outf.Close()
-
-	fStat, err := outf.Stat()
-	if err != nil {
-		return err
-	}
-	header := http.Header{}
-	header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-")
-	url, err := url.Parse(gw + info.Cid)
-	if err != nil {
-		return err
-	}
-	log.Infof("GET %s", url)
-
-	req := http.Request{
-		Method: "GET",
-		URL:    url,
-		Header: header,
-		Close:  true,
-	}
-
-	resp, err := http.DefaultClient.Do(&req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	bar := pb.New64(resp.ContentLength)
-	bar.Units = pb.U_BYTES
-	bar.ShowSpeed = true
-	bar.Start()
-
-	_, err = io.Copy(outf, bar.NewProxyReader(resp.Body))
-
-	bar.Finish()
-
-	return err
-}

paramfetch/proof-params/parameters.json

-{
-  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": {
-    "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9",
-    "digest": "39a9edec3355516674f0d12b926be493",
-    "sector_size": 34359738368
-  },
-  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": {
-    "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX",
-    "digest": "0227ae7df4f2affe529ebafbbc7540ee",
-    "sector_size": 34359738368
-  },
-  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": {
-    "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv",
-    "digest": "991041a354b12c280542741f58c7f2ca",
-    "sector_size": 1024
-  },
-  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": {
-    "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU",
-    "digest": "91fac550e1f9bccab213830bb0c85bd6",
-    "sector_size": 1024
-  },
-  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": {
-    "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4",
-    "digest": "6322eacd2773163ddd51f9ca7d645fc4",
-    "sector_size": 1073741824
-  },
-  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": {
-    "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf",
-    "digest": "0df62745fceac922e3e70847cfc70b52",
-    "sector_size": 1073741824
-  },
-  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": {
-    "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9",
-    "digest": "de9d71e672f286706a1673bd57abdaac",
-    "sector_size": 16777216
-  },
-  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": {
-    "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu",
-    "digest": "7f796d3a0f13499181e44b5eee0cc744",
-    "sector_size": 16777216
-  },
-  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": {
-    "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN",
-    "digest": "151ae0ae183fc141e8c2bebc28e5cc10",
-    "sector_size": 268435456
-  },
-  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": {
-    "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji",
-    "digest": "95eb89588e9d1832aca044c3a13178af",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": {
-    "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX",
-    "digest": "fcd50e2e08a8560a6bb3418e883567ed",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": {
-    "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y",
-    "digest": "872e244d86499fd659082e3bcf3f13e7",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": {
-    "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN",
-    "digest": "0a338b941c5f17946340de5fc95cab30",
-    "sector_size": 34359738368
-  },
-  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": {
-    "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF",
-    "digest": "c752e070a6b7aa8b79aa661a6b600b55",
-    "sector_size": 34359738368
-  },
-  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": {
-    "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186",
-    "digest": "efa407fd09202dffd15799a8518e73d3",
-    "sector_size": 1024
-  },
-  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": {
-    "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG",
-    "digest": "b2f09a0ccb62da28c890d5b881c8dcd2",
-    "sector_size": 1024
-  },
-  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": {
-    "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q",
-    "digest": "b53c1916a63839ec345aa2224e9198b7",
-    "sector_size": 1073741824
-  },
-  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": {
-    "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA",
-    "digest": "20d58a3fae7343481f8298a2dd493dd7",
-    "sector_size": 1073741824
-  },
-  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": {
-    "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU",
-    "digest": "7698426202c7e07b26ef056d31485b3a",
-    "sector_size": 16777216
-  },
-  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": {
-    "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax",
-    "digest": "834408e5c3fce6ec5d1bf64e64cee94e",
-    "sector_size": 16777216
-  }
-}
-

sectorbuilder_test.go

@@ -16,8 +16,8 @@ import (
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log"
 
+	paramfetch "github.com/filecoin-project/go-paramfetch"
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	"github.com/filecoin-project/go-sectorbuilder/paramfetch"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -124,10 +124,19 @@ func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Tim
 	return genCandidates
 }
 
-func TestSealAndVerify(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode")
+// TestDownloadParams exists only so that developers and CI can pre-download
+// Groth parameters and verifying keys before running the tests which rely on
+// those parameters and keys. To do this, run the following command:
+//
+// go test -run=^TestDownloadParams
+//
+func TestDownloadParams(t *testing.T) {
+	if err := paramfetch.GetParams(sectorSize); err != nil {
+		t.Fatalf("%+v", err)
 	}
+}
+
+func TestSealAndVerify(t *testing.T) {
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}
@@ -195,9 +204,6 @@ func TestSealAndVerify(t *testing.T) {
 }
 
 func TestSealPoStNoCommit(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode")
-	}
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}
@@ -262,9 +268,6 @@ func TestSealPoStNoCommit(t *testing.T) {
 }
 
 func TestSealAndVerify2(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode")
-	}
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}