+1323
File diff suppressed because it is too large
Load Diff
+113
@@ -0,0 +1,113 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
)
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
// out or fail are mutually exclusive
|
||||
out time.Duration
|
||||
fail bool
|
||||
}{
|
||||
{in: "10ms", out: 10 * time.Millisecond},
|
||||
{in: "3s", out: 3 * time.Second},
|
||||
{in: "60m", out: 60 * time.Minute},
|
||||
{in: "12h", out: 12 * time.Hour},
|
||||
{in: "7d", out: 168 * time.Hour},
|
||||
|
||||
{in: "", fail: true},
|
||||
{in: "0", fail: true},
|
||||
{in: "7ns", fail: true},
|
||||
{in: "14mo", fail: true},
|
||||
{in: "3.5h", fail: true},
|
||||
{in: "106752d", fail: true}, // overflow
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got, err := parseDuration(tc.in)
|
||||
if !tc.fail && err != nil {
|
||||
t.Errorf("parseDuration(%q) unexpectedly failed: %v", tc.in, err)
|
||||
continue
|
||||
}
|
||||
if tc.fail && err == nil {
|
||||
t.Errorf("parseDuration(%q) did not fail", tc.in)
|
||||
continue
|
||||
}
|
||||
if tc.fail {
|
||||
continue
|
||||
}
|
||||
if got != tc.out {
|
||||
t.Errorf("parseDuration(%q) = %v, want %v", tc.in, got, tc.out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGCPolicy(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
out bigtable.GCPolicy
|
||||
fail bool
|
||||
}{
|
||||
{in: "maxage=1h", out: bigtable.MaxAgePolicy(time.Hour * 1)},
|
||||
{in: "maxversions=2", out: bigtable.MaxVersionsPolicy(int(2))},
|
||||
{in: "maxversions=2 and maxage=1h", out: bigtable.IntersectionPolicy([]bigtable.GCPolicy{bigtable.MaxVersionsPolicy(int(2)), bigtable.MaxAgePolicy(time.Hour * 1)}...)},
|
||||
{in: "maxversions=2 or maxage=1h", out: bigtable.UnionPolicy([]bigtable.GCPolicy{bigtable.MaxVersionsPolicy(int(2)), bigtable.MaxAgePolicy(time.Hour * 1)}...)},
|
||||
|
||||
{in: "maxage=1", fail: true},
|
||||
{in: "maxage = 1h", fail: true},
|
||||
{in: "maxage =1h", fail: true},
|
||||
{in: "maxage= 1h", fail: true},
|
||||
{in: "foomaxage=1h", fail: true},
|
||||
{in: "maxversions=1h", fail: true},
|
||||
{in: "maxversions= 1", fail: true},
|
||||
{in: "maxversions = 1", fail: true},
|
||||
{in: "maxversions =1", fail: true},
|
||||
{in: "barmaxversions=1", fail: true},
|
||||
{in: "maxage = 1h or maxversions=1h", fail: true},
|
||||
{in: "foomaxversions=2 or maxage=1h", fail: true},
|
||||
{in: "maxversions=2 or barmaxage=1h", fail: true},
|
||||
{in: "foomaxversions=2 or barmaxage=1h", fail: true},
|
||||
{in: "maxage = 1h and maxversions=1h", fail: true},
|
||||
{in: "foomaxage=1h and maxversions=1", fail: true},
|
||||
{in: "maxage=1h and barmaxversions=1", fail: true},
|
||||
{in: "foomaxage=1h and barmaxversions=1", fail: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got, err := parseGCPolicy(tc.in)
|
||||
if !tc.fail && err != nil {
|
||||
t.Errorf("parseGCPolicy(%q) unexpectedly failed: %v", tc.in, err)
|
||||
continue
|
||||
}
|
||||
if tc.fail && err == nil {
|
||||
t.Errorf("parseGCPolicy(%q) did not fail", tc.in)
|
||||
continue
|
||||
}
|
||||
if tc.fail {
|
||||
continue
|
||||
}
|
||||
var cmpOpts cmp.Options
|
||||
cmpOpts = append(cmpOpts, cmp.AllowUnexported(bigtable.IntersectionPolicy([]bigtable.GCPolicy{}...)), cmp.AllowUnexported(bigtable.UnionPolicy([]bigtable.GCPolicy{}...)))
|
||||
if !cmp.Equal(got, tc.out, cmpOpts) {
|
||||
t.Errorf("parseGCPolicy(%q) =%v, want %v", tc.in, got, tc.out)
|
||||
}
|
||||
}
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// DO NOT EDIT. THIS IS AUTOMATICALLY GENERATED.
|
||||
// Run "go generate" to regenerate.
|
||||
//go:generate go run cbt.go -o cbtdoc.go doc
|
||||
|
||||
/*
|
||||
Cbt is a tool for doing basic interactions with Cloud Bigtable. To learn how to
|
||||
install the cbt tool, see the
|
||||
[cbt overview](https://cloud.google.com/bigtable/docs/go/cbt-overview).
|
||||
|
||||
Usage:
|
||||
|
||||
cbt [options] command [arguments]
|
||||
|
||||
The commands are:
|
||||
|
||||
count Count rows in a table
|
||||
createinstance Create an instance with an initial cluster
|
||||
createcluster Create a cluster in the configured instance (replication alpha)
|
||||
createfamily Create a column family
|
||||
createtable Create a table
|
||||
updatecluster Update a cluster in the configured instance
|
||||
deleteinstance Deletes an instance
|
||||
deletecluster Deletes a cluster from the configured instance (replication alpha)
|
||||
deletecolumn Delete all cells in a column
|
||||
deletefamily Delete a column family
|
||||
deleterow Delete a row
|
||||
deletetable Delete a table
|
||||
doc Print godoc-suitable documentation for cbt
|
||||
help Print help text
|
||||
listinstances List instances in a project
|
||||
listclusters List instances in an instance
|
||||
lookup Read from a single row
|
||||
ls List tables and column families
|
||||
mddoc Print documentation for cbt in Markdown format
|
||||
read Read rows
|
||||
set Set value of a cell
|
||||
setgcpolicy Set the GC policy for a column family
|
||||
waitforreplication Blocks until all the completed writes have been replicated to all the clusters (replication alpha)
|
||||
version Print the current cbt version
|
||||
|
||||
Use "cbt help <command>" for more information about a command.
|
||||
|
||||
The options are:
|
||||
|
||||
-project string
|
||||
project ID, if unset uses gcloud configured project
|
||||
-instance string
|
||||
Cloud Bigtable instance
|
||||
-creds string
|
||||
if set, use application credentials in this file
|
||||
|
||||
|
||||
Alpha features are not currently available to most Cloud Bigtable customers. The
|
||||
features might be changed in backward-incompatible ways and are not recommended
|
||||
for production use. They are not subject to any SLA or deprecation policy.
|
||||
|
||||
For convenience, values of the -project, -instance, -creds,
|
||||
-admin-endpoint and -data-endpoint flags may be specified in
|
||||
/usr/local/google/home/igorbernstein/.cbtrc in this format:
|
||||
project = my-project-123
|
||||
instance = my-instance
|
||||
creds = path-to-account-key.json
|
||||
admin-endpoint = hostname:port
|
||||
data-endpoint = hostname:port
|
||||
All values are optional, and all will be overridden by flags.
|
||||
|
||||
|
||||
|
||||
Count rows in a table
|
||||
|
||||
Usage:
|
||||
cbt count <table>
|
||||
|
||||
|
||||
|
||||
|
||||
Create an instance with an initial cluster
|
||||
|
||||
Usage:
|
||||
cbt createinstance <instance-id> <display-name> <cluster-id> <zone> <num-nodes> <storage type>
|
||||
instance-id Permanent, unique id for the instance
|
||||
display-name Description of the instance
|
||||
cluster-id Permanent, unique id for the cluster in the instance
|
||||
zone The zone in which to create the cluster
|
||||
num-nodes The number of nodes to create
|
||||
storage-type SSD or HDD
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Create a cluster in the configured instance (replication alpha)
|
||||
|
||||
Usage:
|
||||
cbt createcluster <cluster-id> <zone> <num-nodes> <storage type>
|
||||
cluster-id Permanent, unique id for the cluster in the instance
|
||||
zone The zone in which to create the cluster
|
||||
num-nodes The number of nodes to create
|
||||
storage-type SSD or HDD
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Create a column family
|
||||
|
||||
Usage:
|
||||
cbt createfamily <table> <family>
|
||||
|
||||
|
||||
|
||||
|
||||
Create a table
|
||||
|
||||
Usage:
|
||||
cbt createtable <table> [families=family[:(maxage=<d> | maxversions=<n>)],...] [splits=split,...]
|
||||
families: Column families and their associated GC policies. See "setgcpolicy".
|
||||
Example: families=family1:maxage=1w,family2:maxversions=1
|
||||
splits: Row key to be used to initially split the table
|
||||
|
||||
|
||||
|
||||
|
||||
Update a cluster in the configured instance
|
||||
|
||||
Usage:
|
||||
cbt updatecluster <cluster-id> [num-nodes=num-nodes]
|
||||
cluster-id Permanent, unique id for the cluster in the instance
|
||||
num-nodes The number of nodes to update to
|
||||
|
||||
|
||||
|
||||
|
||||
Deletes an instance
|
||||
|
||||
Usage:
|
||||
cbt deleteinstance <instance>
|
||||
|
||||
|
||||
|
||||
|
||||
Deletes a cluster from the configured instance (replication alpha)
|
||||
|
||||
Usage:
|
||||
cbt deletecluster <cluster>
|
||||
|
||||
|
||||
|
||||
|
||||
Delete all cells in a column
|
||||
|
||||
Usage:
|
||||
cbt deletecolumn <table> <row> <family> <column> [app-profile=<app profile id>]
|
||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Delete a column family
|
||||
|
||||
Usage:
|
||||
cbt deletefamily <table> <family>
|
||||
|
||||
|
||||
|
||||
|
||||
Delete a row
|
||||
|
||||
Usage:
|
||||
cbt deleterow <table> <row> [app-profile=<app profile id>]
|
||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Delete a table
|
||||
|
||||
Usage:
|
||||
cbt deletetable <table>
|
||||
|
||||
|
||||
|
||||
|
||||
Print godoc-suitable documentation for cbt
|
||||
|
||||
Usage:
|
||||
cbt doc
|
||||
|
||||
|
||||
|
||||
|
||||
Print help text
|
||||
|
||||
Usage:
|
||||
cbt help [command]
|
||||
|
||||
|
||||
|
||||
|
||||
List instances in a project
|
||||
|
||||
Usage:
|
||||
cbt listinstances
|
||||
|
||||
|
||||
|
||||
|
||||
List instances in an instance
|
||||
|
||||
Usage:
|
||||
cbt listclusters
|
||||
|
||||
|
||||
|
||||
|
||||
Read from a single row
|
||||
|
||||
Usage:
|
||||
cbt lookup <table> <row> [app-profile=<app profile id>]
|
||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
List tables and column families
|
||||
|
||||
Usage:
|
||||
cbt ls List tables
|
||||
cbt ls <table> List column families in <table>
|
||||
|
||||
|
||||
|
||||
|
||||
Print documentation for cbt in Markdown format
|
||||
|
||||
Usage:
|
||||
cbt mddoc
|
||||
|
||||
|
||||
|
||||
|
||||
Read rows
|
||||
|
||||
Usage:
|
||||
cbt read <table> [start=<row>] [end=<row>] [prefix=<prefix>] [regex=<regex>] [count=<n>] [app-profile=<app profile id>]
|
||||
start=<row> Start reading at this row
|
||||
end=<row> Stop reading before this row
|
||||
prefix=<prefix> Read rows with this prefix
|
||||
regex=<regex> Read rows with keys matching this regex
|
||||
count=<n> Read only this many rows
|
||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Set value of a cell
|
||||
|
||||
Usage:
|
||||
cbt set <table> <row> [app-profile=<app profile id>] family:column=val[@ts] ...
|
||||
app-profile=<app profile id> The app profile id to use for the request (replication alpha)
|
||||
family:column=val[@ts] may be repeated to set multiple cells.
|
||||
|
||||
ts is an optional integer timestamp.
|
||||
If it cannot be parsed, the `@ts` part will be
|
||||
interpreted as part of the value.
|
||||
|
||||
|
||||
|
||||
|
||||
Set the GC policy for a column family
|
||||
|
||||
Usage:
|
||||
cbt setgcpolicy <table> <family> ( maxage=<d> | maxversions=<n> )
|
||||
|
||||
maxage=<d> Maximum timestamp age to preserve (e.g. "1h", "4d")
|
||||
maxversions=<n> Maximum number of versions to preserve
|
||||
|
||||
|
||||
|
||||
|
||||
Blocks until all the completed writes have been replicated to all the clusters (replication alpha)
|
||||
|
||||
Usage:
|
||||
cbt waitforreplication <table>
|
||||
|
||||
|
||||
|
||||
|
||||
Print the current cbt version
|
||||
|
||||
Usage:
|
||||
cbt version
|
||||
|
||||
|
||||
|
||||
|
||||
*/
|
||||
package main
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/*
|
||||
cbtemulator launches the in-memory Cloud Bigtable server on the given address.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"cloud.google.com/go/bigtable/bttest"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
var (
|
||||
host = flag.String("host", "localhost", "the address to bind to on the local machine")
|
||||
port = flag.Int("port", 9000, "the port number to bind to on the local machine")
|
||||
)
|
||||
|
||||
func main() {
|
||||
grpc.EnableTracing = false
|
||||
flag.Parse()
|
||||
srv, err := bttest.NewServer(fmt.Sprintf("%s:%d", *host, *port))
|
||||
if err != nil {
|
||||
log.Fatalf("failed to start emulator: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Cloud Bigtable emulator running on %s\n", srv.Addr)
|
||||
select {}
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
Loadtest does some load testing through the Go client library for Cloud Bigtable.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"cloud.google.com/go/bigtable/internal/cbtconfig"
|
||||
"cloud.google.com/go/bigtable/internal/stat"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/option"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
var (
|
||||
runFor = flag.Duration("run_for", 5*time.Second,
|
||||
"how long to run the load test for; 0 to run forever until SIGTERM")
|
||||
scratchTable = flag.String("scratch_table", "loadtest-scratch", "name of table to use; should not already exist")
|
||||
csvOutput = flag.String("csv_output", "",
|
||||
"output path for statistics in .csv format. If this file already exists it will be overwritten.")
|
||||
poolSize = flag.Int("pool_size", 1, "size of the gRPC connection pool to use for the data client")
|
||||
reqCount = flag.Int("req_count", 100, "number of concurrent requests")
|
||||
|
||||
config *cbtconfig.Config
|
||||
client *bigtable.Client
|
||||
adminClient *bigtable.AdminClient
|
||||
)
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
config, err = cbtconfig.Load()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
config.RegisterFlags()
|
||||
|
||||
flag.Parse()
|
||||
if err := config.CheckFlags(cbtconfig.ProjectAndInstanceRequired); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if config.Creds != "" {
|
||||
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
|
||||
}
|
||||
if flag.NArg() != 0 {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var options []option.ClientOption
|
||||
if *poolSize > 1 {
|
||||
options = append(options,
|
||||
option.WithGRPCConnectionPool(*poolSize),
|
||||
|
||||
// TODO(grpc/grpc-go#1388) using connection pool without WithBlock
|
||||
// can cause RPCs to fail randomly. We can delete this after the issue is fixed.
|
||||
option.WithGRPCDialOption(grpc.WithBlock()))
|
||||
}
|
||||
|
||||
var csvFile *os.File
|
||||
if *csvOutput != "" {
|
||||
csvFile, err = os.Create(*csvOutput)
|
||||
if err != nil {
|
||||
log.Fatalf("creating csv output file: %v", err)
|
||||
}
|
||||
defer csvFile.Close()
|
||||
log.Printf("Writing statistics to %q ...", *csvOutput)
|
||||
}
|
||||
|
||||
log.Printf("Dialing connections...")
|
||||
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance, options...)
|
||||
if err != nil {
|
||||
log.Fatalf("Making bigtable.Client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Making bigtable.AdminClient: %v", err)
|
||||
}
|
||||
defer adminClient.Close()
|
||||
|
||||
// Create a scratch table.
|
||||
log.Printf("Setting up scratch table...")
|
||||
tblConf := bigtable.TableConf{
|
||||
TableID: *scratchTable,
|
||||
Families: map[string]bigtable.GCPolicy{"f": bigtable.MaxVersionsPolicy(1)},
|
||||
}
|
||||
if err := adminClient.CreateTableFromConf(context.Background(), &tblConf); err != nil {
|
||||
log.Fatalf("Making scratch table %q: %v", *scratchTable, err)
|
||||
}
|
||||
// Upon a successful run, delete the table. Don't bother checking for errors.
|
||||
defer adminClient.DeleteTable(context.Background(), *scratchTable)
|
||||
|
||||
// Also delete the table on SIGTERM.
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, os.Interrupt)
|
||||
go func() {
|
||||
s := <-c
|
||||
log.Printf("Caught %v, cleaning scratch table.", s)
|
||||
adminClient.DeleteTable(context.Background(), *scratchTable)
|
||||
os.Exit(1)
|
||||
}()
|
||||
|
||||
log.Printf("Starting load test... (run for %v)", *runFor)
|
||||
tbl := client.Open(*scratchTable)
|
||||
sem := make(chan int, *reqCount) // limit the number of requests happening at once
|
||||
var reads, writes stats
|
||||
stopTime := time.Now().Add(*runFor)
|
||||
var wg sync.WaitGroup
|
||||
for time.Now().Before(stopTime) || *runFor == 0 {
|
||||
sem <- 1
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
ok := true
|
||||
opStart := time.Now()
|
||||
var stats *stats
|
||||
defer func() {
|
||||
stats.Record(ok, time.Since(opStart))
|
||||
}()
|
||||
|
||||
row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows
|
||||
|
||||
switch rand.Intn(10) {
|
||||
default:
|
||||
// read
|
||||
stats = &reads
|
||||
_, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
|
||||
if err != nil {
|
||||
log.Printf("Error doing read: %v", err)
|
||||
ok = false
|
||||
}
|
||||
case 0, 1, 2, 3, 4:
|
||||
// write
|
||||
stats = &writes
|
||||
mut := bigtable.NewMutation()
|
||||
mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write
|
||||
if err := tbl.Apply(context.Background(), row, mut); err != nil {
|
||||
log.Printf("Error doing mutation: %v", err)
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
readsAgg := stat.NewAggregate("reads", reads.ds, reads.tries-reads.ok)
|
||||
writesAgg := stat.NewAggregate("writes", writes.ds, writes.tries-writes.ok)
|
||||
log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, readsAgg)
|
||||
log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, writesAgg)
|
||||
|
||||
if csvFile != nil {
|
||||
stat.WriteCSV([]*stat.Aggregate{readsAgg, writesAgg}, csvFile)
|
||||
}
|
||||
}
|
||||
|
||||
var allStats int64 // atomic
|
||||
|
||||
type stats struct {
|
||||
mu sync.Mutex
|
||||
tries, ok int
|
||||
ds []time.Duration
|
||||
}
|
||||
|
||||
func (s *stats) Record(ok bool, d time.Duration) {
|
||||
s.mu.Lock()
|
||||
s.tries++
|
||||
if ok {
|
||||
s.ok++
|
||||
}
|
||||
s.ds = append(s.ds, d)
|
||||
s.mu.Unlock()
|
||||
|
||||
if n := atomic.AddInt64(&allStats, 1); n%1000 == 0 {
|
||||
log.Printf("Progress: done %d ops", n)
|
||||
}
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
Copyright 2016 Google Inc. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
Scantest does scan-related load testing against Cloud Bigtable. The logic here
|
||||
mimics a similar test written using the Java client.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"cloud.google.com/go/bigtable/internal/cbtconfig"
|
||||
"cloud.google.com/go/bigtable/internal/stat"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var (
|
||||
runFor = flag.Duration("run_for", 5*time.Second, "how long to run the load test for")
|
||||
numScans = flag.Int("concurrent_scans", 1, "number of concurrent scans")
|
||||
rowLimit = flag.Int("row_limit", 10000, "max number of records per scan")
|
||||
|
||||
config *cbtconfig.Config
|
||||
client *bigtable.Client
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Usage = func() {
|
||||
fmt.Printf("Usage: scantest [options] <table_name>\n\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
var err error
|
||||
config, err = cbtconfig.Load()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
config.RegisterFlags()
|
||||
|
||||
flag.Parse()
|
||||
if err := config.CheckFlags(cbtconfig.ProjectAndInstanceRequired); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if config.Creds != "" {
|
||||
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
|
||||
}
|
||||
if flag.NArg() != 1 {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
table := flag.Arg(0)
|
||||
|
||||
log.Printf("Dialing connections...")
|
||||
client, err = bigtable.NewClient(context.Background(), config.Project, config.Instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Making bigtable.Client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
log.Printf("Starting scan test... (run for %v)", *runFor)
|
||||
tbl := client.Open(table)
|
||||
sem := make(chan int, *numScans) // limit the number of requests happening at once
|
||||
var scans stats
|
||||
|
||||
stopTime := time.Now().Add(*runFor)
|
||||
var wg sync.WaitGroup
|
||||
for time.Now().Before(stopTime) {
|
||||
sem <- 1
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
|
||||
ok := true
|
||||
opStart := time.Now()
|
||||
defer func() {
|
||||
scans.Record(ok, time.Since(opStart))
|
||||
}()
|
||||
|
||||
// Start at a random row key
|
||||
key := fmt.Sprintf("user%d", rand.Int63())
|
||||
limit := bigtable.LimitRows(int64(*rowLimit))
|
||||
noop := func(bigtable.Row) bool { return true }
|
||||
if err := tbl.ReadRows(context.Background(), bigtable.NewRange(key, ""), noop, limit); err != nil {
|
||||
log.Printf("Error during scan: %v", err)
|
||||
ok = false
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
agg := stat.NewAggregate("scans", scans.ds, scans.tries-scans.ok)
|
||||
log.Printf("Scans (%d ok / %d tries):\nscan times:\n%v\nthroughput (rows/second):\n%v",
|
||||
scans.ok, scans.tries, agg, throughputString(agg))
|
||||
}
|
||||
|
||||
func throughputString(agg *stat.Aggregate) string {
|
||||
var buf bytes.Buffer
|
||||
tw := tabwriter.NewWriter(&buf, 0, 0, 1, ' ', 0) // one-space padding
|
||||
rowLimitF := float64(*rowLimit)
|
||||
fmt.Fprintf(
|
||||
tw,
|
||||
"min:\t%.2f\nmedian:\t%.2f\nmax:\t%.2f\n",
|
||||
rowLimitF/agg.Max.Seconds(),
|
||||
rowLimitF/agg.Median.Seconds(),
|
||||
rowLimitF/agg.Min.Seconds())
|
||||
tw.Flush()
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
var allStats int64 // atomic
|
||||
|
||||
type stats struct {
|
||||
mu sync.Mutex
|
||||
tries, ok int
|
||||
ds []time.Duration
|
||||
}
|
||||
|
||||
func (s *stats) Record(ok bool, d time.Duration) {
|
||||
s.mu.Lock()
|
||||
s.tries++
|
||||
if ok {
|
||||
s.ok++
|
||||
}
|
||||
s.ds = append(s.ds, d)
|
||||
s.mu.Unlock()
|
||||
|
||||
if n := atomic.AddInt64(&allStats, 1); n%1000 == 0 {
|
||||
log.Printf("Progress: done %d ops", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user