diff --git a/alltest.sh b/alltest.sh index 31421cda0..e362edbfb 100755 --- a/alltest.sh +++ b/alltest.sh @@ -15,7 +15,7 @@ main() { # some benchmarks go test -tags "$UNITTESTTAGS" -bench=^BenchmarkPersistentCaller_Call$ -run ^$ ./rpc/ bash cleanupDB.sh || true - go test -tags "$UNITTESTTAGS" -bench=^BenchmarkMinerTwo$ -benchtime=5s -run ^$ ./cmd/cql-minerd/ + go test -tags "$UNITTESTTAGS" -bench=^BenchmarkMiner$ -benchtime=5s -run ^$ ./cmd/cql-minerd/ -bench-miner-count=2 bash cleanupDB.sh || true } diff --git a/cmd/cql-minerd/bench.sh b/cmd/cql-minerd/bench.sh index 837b8d9eb..e9d29d6e2 100755 --- a/cmd/cql-minerd/bench.sh +++ b/cmd/cql-minerd/bench.sh @@ -1,17 +1,27 @@ -#!/bin/bash +#! /usr/bin/env bash +set -euo pipefail -make -C ../../ clean && \ -make -C ../../ use_all_cores && \ -go test -bench=^BenchmarkSQLite$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerOne$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerOneNoSign$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerTwo$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerTwoNoSign$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerThree$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerThreeNoSign$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerOneWithEventualConsistency$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerOneNoSignWithEventualConsistency$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerTwoWithEventualConsistency$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerTwoNoSignWithEventualConsistency$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerThreeWithEventualConsistency$ -benchtime=10s -run ^$ && \ -go test -bench=^BenchmarkMinerThreeNoSignWithEventualConsistency$ -benchtime=10s -run ^$ + +main() { + local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" + make -C "$wd" clean + make -C "$wd" use_all_cores + + local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd" + #go test -bench=^BenchmarkSQLite$ -benchtime=10s -run=^$ "$pkg" + + local flags=( + "-bench=^BenchmarkMiner$" + "-benchtime=10s" + "-run=^$" + ) + local i + for ((i=1; i<=3; i++)); do + go test "${flags[@]}" "$pkg" -bench-miner-count=$i + go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-bypass-signature + go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-eventual-consistency + go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-bypass-signature -bench-eventual-consistency + done +} + +main "$@" diff --git a/cmd/cql-minerd/benchCustom.sh b/cmd/cql-minerd/benchCustom.sh index 6590d3b74..bdc7dc054 100755 --- a/cmd/cql-minerd/benchCustom.sh +++ b/cmd/cql-minerd/benchCustom.sh @@ -1,12 +1,29 @@ -#!/bin/bash - -make -C ../../ clean && \ -make -C ../../ use_all_cores -export miner_conf_dir=$PWD/../../test/bench_testnet/node_c -go test -bench=^BenchmarkCustomMiner1$ -benchtime=10s -run ^$ |tee custom_miner.log -go test -bench=^BenchmarkCustomMiner2$ -benchtime=10s -run ^$ |tee -a custom_miner.log -go test -bench=^BenchmarkCustomMiner3$ -benchtime=10s -run ^$ |tee -a custom_miner.log - -go test -cpu=1 -bench=^BenchmarkCustomMiner1$ -benchtime=10s -run ^$ |tee -a custom_miner.log -go test -cpu=1 -bench=^BenchmarkCustomMiner2$ -benchtime=10s -run ^$ |tee -a custom_miner.log -go test -cpu=1 -bench=^BenchmarkCustomMiner3$ -benchtime=10s -run ^$ |tee -a custom_miner.log +#! /usr/bin/env bash +set -euo pipefail + +main() { + local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" + make -C "$wd" clean + make -C "$wd" use_all_cores + + rm -f custom_miner.log + touch custom_miner.log + + local flags=( + "-bench=^BenchmarkCustomMiner$" + "-benchtime=10s" + "-run=^$" + ) + local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd" + local i subflags + for ((i=1; i<=3; i++)); do + subflags=( + "-bench-miner-config-dir=$wd/test/bench_testnet/node_c" + "-bench-miner-count=$i" + ) + go test "${flags[@]}" "$pkg" "${subflags[@]}" | tee -a custom_miner.log + go test -cpu=1 "${flags[@]}" "$pkg" "${subflags[@]}" | tee -a custom_miner.log + done +} + +main "$@" diff --git a/cmd/cql-minerd/benchGNTE.sh b/cmd/cql-minerd/benchGNTE.sh index c11f14ce5..3e0657123 100755 --- a/cmd/cql-minerd/benchGNTE.sh +++ b/cmd/cql-minerd/benchGNTE.sh @@ -1,30 +1,45 @@ -#!/bin/bash +#! /usr/bin/env bash +set -euo pipefail -param=$1 +declare pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd" +declare flags=( + "-bench=^BenchmarkMinerGNTE$" + "-benchtime=10s" + "-run=^$" +) -#make -C ../../ clean && \ -#make -C ../../ use_all_cores && \ +fast() { + echo "Fast benchmarking with flags: $@" + go test "${flags[@]}" "$pkg" "$@" | tee -a gnte.log + go test "${flags[@]}" "$pkg" "$@" -bench-miner-count=2 | tee -a gnte.log + go test -cpu=1 "${flags[@]}" "$pkg" "$@" -bench-miner-count=2 | tee -a gnte.log +} -if [ "fast" == "$param" ]; then - go test -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee gnte.log - go test -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=1 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log -else - go test -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee gnte.log - go test -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log +full() { + echo "Full benchmarking with flags: $@" + local cpus=("" 4 1) counts=(1 2 3 4 8) + local cpu count caseflags + for cpu in "${cpus[@]}"; do + if [[ -z $cpu ]]; then + caseflags=("${flags[@]}") + else + caseflags=("-cpu=$cpu" "${flags[@]}") + fi + for count in "${counts[@]}"; do + go test "${caseflags[@]}" "$pkg" "$@" -bench-miner-count=$count | tee -a gnte.log + done + done +} - go test -cpu=4 -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=4 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=4 -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=4 -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=4 -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log +main() { + rm -f gnte.log + touch gnte.log + if [[ $# -gt 0 && $1 = "fast" ]]; then + fast + else + full + full -bench-eventual-consistency + fi +} - go test -cpu=1 -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=1 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=1 -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=1 -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log - go test -cpu=1 -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log -fi +main "$@" diff --git a/cmd/cql-minerd/benchTestnet.sh b/cmd/cql-minerd/benchTestnet.sh index 81e0c2912..6e2c86943 100755 --- a/cmd/cql-minerd/benchTestnet.sh +++ b/cmd/cql-minerd/benchTestnet.sh @@ -1,11 +1,32 @@ -#!/bin/bash +#! /usr/bin/env bash +set -euo pipefail -make -C ../../ clean && \ -make -C ../../ use_all_cores -go test -bench=^BenchmarkTestnetMiner1$ -benchtime=10s -run ^$ |tee testnet.log -go test -bench=^BenchmarkTestnetMiner2$ -benchtime=10s -run ^$ |tee -a testnet.log -go test -bench=^BenchmarkTestnetMiner3$ -benchtime=10s -run ^$ |tee -a testnet.log +main() { + local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" + make -C "$wd" clean + make -C "$wd" use_all_cores -go test -cpu=1 -bench=^BenchmarkTestnetMiner1$ -benchtime=10s -run ^$ |tee -a testnet.log -go test -cpu=1 -bench=^BenchmarkTestnetMiner2$ -benchtime=10s -run ^$ |tee -a testnet.log -go test -cpu=1 -bench=^BenchmarkTestnetMiner3$ -benchtime=10s -run ^$ |tee -a testnet.log + rm -f testnet.log + touch testnet.log + + local flags=( + "-bench=^BenchmarkTestnetMiner$" + "-benchtime=10s" + "-run=^$" + ) + local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd" + local cpus=("" 1) counts=(1 2 3) + local cpu count caseflags + for cpu in "${cpus[@]}"; do + if [[ -z $cpu ]]; then + caseflags=("${flags[@]}") + else + caseflags=("-cpu=$cpu" "${flags[@]}") + fi + for count in "${counts[@]}"; do + go test "${caseflags[@]}" "$pkg" -bench-miner-count=$count | tee -a testnet.log + done + done +} + +main "$@" diff --git a/cmd/cql-minerd/integration_test.go b/cmd/cql-minerd/integration_test.go index ff818f234..a99a40984 100644 --- a/cmd/cql-minerd/integration_test.go +++ b/cmd/cql-minerd/integration_test.go @@ -21,6 +21,7 @@ package main import ( "context" "database/sql" + "flag" "fmt" "io/ioutil" "math/rand" @@ -28,6 +29,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "sync" "sync/atomic" "syscall" @@ -36,7 +38,6 @@ import ( sqlite3 "github.com/CovenantSQL/go-sqlite3-encrypt" . "github.com/smartystreets/goconvey/convey" - yaml "gopkg.in/yaml.v2" "github.com/CovenantSQL/CovenantSQL/client" "github.com/CovenantSQL/CovenantSQL/conf" @@ -61,11 +62,28 @@ var ( logDir = FJ(testWorkingDir, "./log/") testGasPrice uint64 = 1 testAdvancePayment uint64 = 20000000 -) -var nodeCmds []*utils.CMD + nodeCmds []*utils.CMD + + FJ = filepath.Join -var FJ = filepath.Join + // Benchmark flags + benchMinerCount int + benchBypassSignature bool + benchEventualConsistency bool + benchMinerConfigDir string +) + +func init() { + flag.IntVar(&benchMinerCount, "bench-miner-count", 1, + "Benchmark miner count.") + flag.BoolVar(&benchBypassSignature, "bench-bypass-signature", false, + "Benchmark bypassing signature.") + flag.BoolVar(&benchEventualConsistency, "bench-eventual-consistency", false, + "Benchmark with eventaul consistency.") + flag.StringVar(&benchMinerConfigDir, "bench-miner-config-dir", "", + "Benchmark custome miner config directory.") +} func startNodes() { ctx := context.Background() @@ -631,6 +649,19 @@ func cleanBenchTable(db *sql.DB) { So(err, ShouldBeNil) } +func makeBenchName(trailings ...string) string { + var parts = make([]string, 0, 3+len(trailings)) + parts = append(parts, fmt.Sprintf("%dMiner", benchMinerCount)) + if benchBypassSignature { + parts = append(parts, "BypassSignature") + } + if benchEventualConsistency { + parts = append(parts, "EventualConsistency") + } + parts = append(parts, trailings...) + return strings.Join(parts, "_") +} + func benchDB(b *testing.B, db *sql.DB, createDB bool) { var err error if createDB { @@ -642,7 +673,7 @@ func benchDB(b *testing.B, db *sql.DB, createDB bool) { var i int64 i = -1 - b.Run("benchmark INSERT", func(b *testing.B) { + b.Run(makeBenchName("INSERT"), func(b *testing.B) { b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -687,7 +718,7 @@ func benchDB(b *testing.B, db *sql.DB, createDB bool) { } log.Warnf("row Count: %v", count) - b.Run("benchmark SELECT", func(b *testing.B) { + b.Run(makeBenchName("SELECT"), func(b *testing.B) { b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { @@ -732,11 +763,11 @@ func benchDB(b *testing.B, db *sql.DB, createDB bool) { So(err, ShouldBeNil) } -func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualConsistency bool) { - log.Warnf("benchmark for %d Miners, BypassSignature: %v", minerCount, bypassSign) - asymmetric.BypassSignature = bypassSign +func benchMiner(b *testing.B, minerCount uint16) { + log.Warnf("benchmark for %d Miners, BypassSignature: %v", minerCount, benchBypassSignature) + asymmetric.BypassSignature = benchBypassSignature if minerCount > 0 { - startNodesProfile(bypassSign) + startNodesProfile(benchBypassSignature) utils.WaitToConnect(context.Background(), "127.0.0.1", []int{ 2144, 2145, @@ -770,8 +801,8 @@ func benchMiner(b *testing.B, minerCount uint16, bypassSign bool, useEventualCon // create meta := client.ResourceMeta{ ResourceMeta: types.ResourceMeta{ - Node: minerCount, - UseEventualConsistency: useEventualConsistency, + Node: minerCount, + UseEventualConsistency: benchEventualConsistency, }, } // wait for chain service @@ -919,172 +950,32 @@ func benchOutsideMinerWithTargetMinerList( benchDB(b, db, minerCount > 0) } -func BenchmarkMinerOneNoSign(b *testing.B) { - Convey("bench single node", b, func() { - benchMiner(b, 1, true, false) - }) -} - -func BenchmarkMinerTwoNoSign(b *testing.B) { - Convey("bench two node", b, func() { - benchMiner(b, 2, true, false) - }) -} - -func BenchmarkMinerThreeNoSign(b *testing.B) { - Convey("bench three node", b, func() { - benchMiner(b, 3, true, false) - }) -} - -func BenchmarkMinerOne(b *testing.B) { - Convey("bench single node", b, func() { - benchMiner(b, 1, false, false) - }) -} - -func BenchmarkMinerTwo(b *testing.B) { - Convey("bench two node", b, func() { - benchMiner(b, 2, false, false) - }) -} - -func BenchmarkMinerThree(b *testing.B) { - Convey("bench three node", b, func() { - benchMiner(b, 3, false, false) - }) -} - -func BenchmarkMinerOneNoSignWithEventualConsistency(b *testing.B) { - Convey("bench single node", b, func() { - benchMiner(b, 1, true, true) - }) -} - -func BenchmarkMinerTwoNoSignWithEventualConsistency(b *testing.B) { - Convey("bench two node", b, func() { - benchMiner(b, 2, true, true) - }) -} - -func BenchmarkMinerThreeNoSignWithEventualConsistency(b *testing.B) { - Convey("bench three node", b, func() { - benchMiner(b, 3, true, true) - }) -} - -func BenchmarkMinerOneWithEventualConsistency(b *testing.B) { - Convey("bench single node", b, func() { - benchMiner(b, 1, false, true) - }) -} - -func BenchmarkMinerTwoWithEventualConsistency(b *testing.B) { - Convey("bench two node", b, func() { - benchMiner(b, 2, false, true) - }) -} - -func BenchmarkMinerThreeWithEventualConsistency(b *testing.B) { - Convey("bench three node", b, func() { - benchMiner(b, 3, false, true) - }) -} - func BenchmarkClientOnly(b *testing.B) { Convey("bench three node", b, func() { - benchMiner(b, 0, false, false) - }) -} - -func BenchmarkMinerGNTE1(b *testing.B) { - Convey("bench GNTE one node", b, func() { - benchOutsideMiner(b, 1, gnteConfDir) - }) -} - -func BenchmarkMinerGNTE2(b *testing.B) { - Convey("bench GNTE two node", b, func() { - benchOutsideMiner(b, 2, gnteConfDir) - }) -} - -func BenchmarkMinerGNTE3(b *testing.B) { - Convey("bench GNTE three node", b, func() { - benchOutsideMiner(b, 3, gnteConfDir) - }) -} - -func BenchmarkMinerGNTE4(b *testing.B) { - Convey("bench GNTE three node", b, func() { - benchOutsideMiner(b, 4, gnteConfDir) - }) -} - -func BenchmarkMinerGNTE8(b *testing.B) { - Convey("bench GNTE three node", b, func() { - benchOutsideMiner(b, 8, gnteConfDir) - }) -} - -func BenchmarkTestnetMiner1(b *testing.B) { - Convey("bench testnet one node", b, func() { - benchOutsideMiner(b, 1, testnetConfDir) - }) -} - -func BenchmarkTestnetMiner2(b *testing.B) { - Convey("bench testnet one node", b, func() { - benchOutsideMiner(b, 2, testnetConfDir) - }) -} - -func BenchmarkTestnetTargetMiner2(b *testing.B) { - var ( - err error - // Public keys of miners for test - publicKeys = []string{ - "0235abfb93031df7bf776332c510a862e48e81eebea76f5e165406af8fec5215d6", - "03aec5337c0a58b8eff96f8ab30518830ad8e329c74bb30b38901a9395c72340f8", - } - ) - Convey("bench testnet one node", b, func() { - var ( - pubKey asymmetric.PublicKey - addr proto.AccountAddress - targetMiners = make([]proto.AccountAddress, len(publicKeys)) - ) - for i, v := range publicKeys { - err = yaml.Unmarshal([]byte(v), &pubKey) - So(err, ShouldBeNil) - addr, err = crypto.PubKeyHash(&pubKey) - So(err, ShouldBeNil) - targetMiners[i] = addr - } - benchOutsideMinerWithTargetMinerList(b, 2, targetMiners, testnetConfDir) + benchMiner(b, 0) }) } -func BenchmarkTestnetMiner3(b *testing.B) { - Convey("bench testnet one node", b, func() { - benchOutsideMiner(b, 3, testnetConfDir) +func BenchmarkMiner(b *testing.B) { + Convey(fmt.Sprintf("bench %d node(s)", benchMinerCount), b, func() { + benchMiner(b, uint16(benchMinerCount)) }) } -func BenchmarkCustomMiner1(b *testing.B) { - Convey("bench custom one node", b, func() { - benchOutsideMiner(b, 1, os.Getenv("miner_conf_dir")) +func BenchmarkMinerGNTE(b *testing.B) { + Convey(fmt.Sprintf("bench GNTE %d node(s)", benchMinerCount), b, func() { + benchOutsideMiner(b, uint16(benchMinerCount), gnteConfDir) }) } -func BenchmarkCustomMiner2(b *testing.B) { - Convey("bench custom one node", b, func() { - benchOutsideMiner(b, 2, os.Getenv("miner_conf_dir")) +func BenchmarkTestnetMiner(b *testing.B) { + Convey(fmt.Sprintf("bench testnet %d node(s)", benchMinerCount), b, func() { + benchOutsideMiner(b, uint16(benchMinerCount), testnetConfDir) }) } -func BenchmarkCustomMiner3(b *testing.B) { - Convey("bench custom one node", b, func() { - benchOutsideMiner(b, 3, os.Getenv("miner_conf_dir")) +func BenchmarkCustomMiner(b *testing.B) { + Convey(fmt.Sprintf("bench custom %d node(s)", benchMinerCount), b, func() { + benchOutsideMiner(b, uint16(benchMinerCount), benchMinerConfigDir) }) } diff --git a/cmd/cql-minerd/main_test.go b/cmd/cql-minerd/main_test.go index 03a7b6494..9d445cf05 100644 --- a/cmd/cql-minerd/main_test.go +++ b/cmd/cql-minerd/main_test.go @@ -18,9 +18,13 @@ package main -import "testing" +import ( + "flag" + "testing" +) func TestMain(m *testing.M) { + flag.Parse() defer m.Run() main() }