+101
@@ -0,0 +1,101 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cloud.google.com/go/profiler"
|
||||
"compress/gzip"
|
||||
"flag"
|
||||
"log"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
service = flag.String("service", "", "service name")
|
||||
mutexProfiling = flag.Bool("mutex_profiling", false, "enable mutex profiling")
|
||||
duration = flag.Int("duration", 600, "duration of the benchmark in seconds")
|
||||
apiAddr = flag.String("api_address", "", "API address of the profiler (e.g. 'cloudprofiler.googleapis.com:443')")
|
||||
)
|
||||
|
||||
// busywork continuously generates 1MiB of random data and compresses it
|
||||
// throwing away the result.
|
||||
func busywork(mu *sync.Mutex) {
|
||||
ticker := time.NewTicker(time.Duration(*duration) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
return
|
||||
default:
|
||||
mu.Lock()
|
||||
busyworkOnce()
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func busyworkOnce() {
|
||||
data := make([]byte, 1024*1024)
|
||||
rand.Read(data)
|
||||
|
||||
var b bytes.Buffer
|
||||
gz := gzip.NewWriter(&b)
|
||||
if _, err := gz.Write(data); err != nil {
|
||||
log.Printf("Failed to write to gzip stream: %v", err)
|
||||
return
|
||||
}
|
||||
if err := gz.Flush(); err != nil {
|
||||
log.Printf("Failed to flush to gzip stream: %v", err)
|
||||
return
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
log.Printf("Failed to close gzip stream: %v", err)
|
||||
}
|
||||
// Throw away the result.
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if *service == "" {
|
||||
log.Print("Service name must be configured using --service flag.")
|
||||
} else if err := profiler.Start(
|
||||
profiler.Config{
|
||||
Service: *service,
|
||||
MutexProfiling: *mutexProfiling,
|
||||
DebugLogging: true,
|
||||
APIAddr: *apiAddr,
|
||||
}); err != nil {
|
||||
log.Printf("Failed to start the profiler: %v", err)
|
||||
} else {
|
||||
mu := new(sync.Mutex)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(5)
|
||||
for i := 0; i < 5; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
busywork(mu)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
log.Printf("busybench finished profiling.")
|
||||
// Do not exit, since the pod in the GKE test is set to always restart.
|
||||
select {}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Fail on any error.
|
||||
set -eo pipefail
|
||||
|
||||
# Display commands being run.
|
||||
set -x
|
||||
|
||||
cd git/gocloud
|
||||
COMMIT=$(git rev-parse HEAD)
|
||||
|
||||
# Set $GOPATH
|
||||
export GOPATH="$HOME/go"
|
||||
GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
|
||||
mkdir -p $GOCLOUD_HOME
|
||||
|
||||
# Move code into $GOPATH and get dependencies
|
||||
cp -R ./* $GOCLOUD_HOME
|
||||
cd $GOCLOUD_HOME/internal/kokoro
|
||||
# Don't print out encryption keys, etc
|
||||
set +x
|
||||
key=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_key")
|
||||
iv=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_iv")
|
||||
pass=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_pass")
|
||||
|
||||
openssl aes-256-cbc -K $key -iv $iv -pass pass:$pass -in kokoro-key.json.enc -out key.json -d
|
||||
set -x
|
||||
|
||||
export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/key.json"
|
||||
export GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762"
|
||||
export GCLOUD_TESTS_GOLANG_ZONE="us-west1-a"
|
||||
export GCLOUD_TESTS_GOLANG_BUCKET="dulcet-port-762-go-cloud-profiler-test"
|
||||
|
||||
cd $GOCLOUD_HOME/profiler
|
||||
go get -t -tags=integration .
|
||||
go test -timeout=60m -parallel=5 -tags=integration -run TestAgentIntegration -commit="$COMMIT"
|
||||
+266
@@ -0,0 +1,266 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build integration,go1.7
|
||||
|
||||
package profiler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/profiler/proftest"
|
||||
"golang.org/x/net/context"
|
||||
"golang.org/x/oauth2/google"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
)
|
||||
|
||||
var (
|
||||
commit = flag.String("commit", "", "git commit to test")
|
||||
runID = time.Now().Unix()
|
||||
)
|
||||
|
||||
const (
|
||||
cloudScope = "https://www.googleapis.com/auth/cloud-platform"
|
||||
benchFinishString = "busybench finished profiling"
|
||||
)
|
||||
|
||||
const startupTemplate = `
|
||||
#! /bin/bash
|
||||
|
||||
(
|
||||
# Shut down the VM in 5 minutes after this script exits
|
||||
# to stop accounting the VM for billing and cores quota.
|
||||
trap "sleep 300 && poweroff" EXIT
|
||||
|
||||
# Fail on any error.
|
||||
set -eo pipefail
|
||||
|
||||
# Display commands being run.
|
||||
set -x
|
||||
|
||||
# Install git
|
||||
apt-get update >/dev/null
|
||||
apt-get -y -q install git >/dev/null
|
||||
|
||||
# Install desired Go version
|
||||
mkdir -p /tmp/bin
|
||||
curl -sL -o /tmp/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
|
||||
chmod +x /tmp/bin/gimme
|
||||
export PATH=$PATH:/tmp/bin
|
||||
|
||||
eval "$(gimme {{.GoVersion}})"
|
||||
|
||||
# Set $GOPATH
|
||||
export GOPATH="$HOME/go"
|
||||
|
||||
export GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
|
||||
mkdir -p $GOCLOUD_HOME
|
||||
|
||||
# Install agent
|
||||
git clone https://code.googlesource.com/gocloud $GOCLOUD_HOME >/dev/null
|
||||
|
||||
cd $GOCLOUD_HOME/profiler/busybench
|
||||
git reset --hard {{.Commit}}
|
||||
go get -v >/dev/null
|
||||
|
||||
# Run benchmark with agent
|
||||
go run busybench.go --service="{{.Service}}" --mutex_profiling="{{.MutexProfiling}}"
|
||||
|
||||
# Write output to serial port 2 with timestamp.
|
||||
) 2>&1 | while read line; do echo "$(date): ${line}"; done >/dev/ttyS1
|
||||
`
|
||||
|
||||
const dockerfileFmt = `FROM golang
|
||||
RUN git clone https://code.googlesource.com/gocloud /go/src/cloud.google.com/go \
|
||||
&& cd /go/src/cloud.google.com/go/profiler/busybench && git reset --hard %s \
|
||||
&& go get -v && go install -v
|
||||
CMD ["busybench", "--service", "%s"]
|
||||
`
|
||||
|
||||
type goGCETestCase struct {
|
||||
proftest.InstanceConfig
|
||||
name string
|
||||
goVersion string
|
||||
mutexProfiling bool
|
||||
wantProfileTypes []string
|
||||
}
|
||||
|
||||
func (tc *goGCETestCase) initializeStartupScript(template *template.Template) error {
|
||||
var buf bytes.Buffer
|
||||
err := template.Execute(&buf,
|
||||
struct {
|
||||
Service string
|
||||
GoVersion string
|
||||
Commit string
|
||||
MutexProfiling bool
|
||||
}{
|
||||
Service: tc.name,
|
||||
GoVersion: tc.goVersion,
|
||||
Commit: *commit,
|
||||
MutexProfiling: tc.mutexProfiling,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
|
||||
}
|
||||
tc.StartupScript = buf.String()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAgentIntegration(t *testing.T) {
|
||||
projectID := os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
|
||||
if projectID == "" {
|
||||
t.Fatalf("Getenv(GCLOUD_TESTS_GOLANG_PROJECT_ID) got empty string")
|
||||
}
|
||||
|
||||
zone := os.Getenv("GCLOUD_TESTS_GOLANG_ZONE")
|
||||
if zone == "" {
|
||||
t.Fatalf("Getenv(GCLOUD_TESTS_GOLANG_ZONE) got empty string")
|
||||
}
|
||||
|
||||
if *commit == "" {
|
||||
t.Fatal("commit flag is not set")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
client, err := google.DefaultClient(ctx, cloudScope)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get default client: %v", err)
|
||||
}
|
||||
|
||||
computeService, err := compute.New(client)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to initialize compute service: %v", err)
|
||||
}
|
||||
|
||||
template, err := template.New("startupScript").Parse(startupTemplate)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse startup script template: %v", err)
|
||||
}
|
||||
|
||||
tr := proftest.TestRunner{
|
||||
Client: client,
|
||||
}
|
||||
|
||||
gceTr := proftest.GCETestRunner{
|
||||
TestRunner: tr,
|
||||
ComputeService: computeService,
|
||||
}
|
||||
|
||||
testcases := []goGCETestCase{
|
||||
{
|
||||
InstanceConfig: proftest.InstanceConfig{
|
||||
ProjectID: projectID,
|
||||
Zone: zone,
|
||||
Name: fmt.Sprintf("profiler-test-go110-%d", runID),
|
||||
MachineType: "n1-standard-1",
|
||||
},
|
||||
name: fmt.Sprintf("profiler-test-go110-%d-gce", runID),
|
||||
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
|
||||
goVersion: "1.10",
|
||||
mutexProfiling: true,
|
||||
},
|
||||
{
|
||||
InstanceConfig: proftest.InstanceConfig{
|
||||
ProjectID: projectID,
|
||||
Zone: zone,
|
||||
Name: fmt.Sprintf("profiler-test-go19-%d", runID),
|
||||
MachineType: "n1-standard-1",
|
||||
},
|
||||
name: fmt.Sprintf("profiler-test-go19-%d-gce", runID),
|
||||
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
|
||||
goVersion: "1.9",
|
||||
mutexProfiling: true,
|
||||
},
|
||||
{
|
||||
InstanceConfig: proftest.InstanceConfig{
|
||||
ProjectID: projectID,
|
||||
Zone: zone,
|
||||
Name: fmt.Sprintf("profiler-test-go18-%d", runID),
|
||||
MachineType: "n1-standard-1",
|
||||
},
|
||||
name: fmt.Sprintf("profiler-test-go18-%d-gce", runID),
|
||||
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
|
||||
goVersion: "1.8",
|
||||
mutexProfiling: true,
|
||||
},
|
||||
{
|
||||
InstanceConfig: proftest.InstanceConfig{
|
||||
ProjectID: projectID,
|
||||
Zone: zone,
|
||||
Name: fmt.Sprintf("profiler-test-go17-%d", runID),
|
||||
MachineType: "n1-standard-1",
|
||||
},
|
||||
name: fmt.Sprintf("profiler-test-go17-%d-gce", runID),
|
||||
wantProfileTypes: []string{"CPU", "HEAP", "THREADS"},
|
||||
goVersion: "1.7",
|
||||
},
|
||||
{
|
||||
InstanceConfig: proftest.InstanceConfig{
|
||||
ProjectID: projectID,
|
||||
Zone: zone,
|
||||
Name: fmt.Sprintf("profiler-test-go16-%d", runID),
|
||||
MachineType: "n1-standard-1",
|
||||
},
|
||||
name: fmt.Sprintf("profiler-test-go16-%d-gce", runID),
|
||||
wantProfileTypes: []string{"CPU", "HEAP", "THREADS"},
|
||||
goVersion: "1.6",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testcases {
|
||||
tc := tc // capture range variable
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if err := tc.initializeStartupScript(template); err != nil {
|
||||
t.Fatalf("failed to initialize startup script")
|
||||
}
|
||||
|
||||
if err := gceTr.StartInstance(ctx, &tc.InstanceConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutCtx, cancel := context.WithTimeout(ctx, time.Minute*25)
|
||||
defer cancel()
|
||||
if err := gceTr.PollForSerialOutput(timeoutCtx, &tc.InstanceConfig, benchFinishString); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
timeNow := time.Now()
|
||||
endTime := timeNow.Format(time.RFC3339)
|
||||
startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
|
||||
for _, pType := range tc.wantProfileTypes {
|
||||
pr, err := tr.QueryProfiles(tc.ProjectID, tc.name, startTime, endTime, pType)
|
||||
if err != nil {
|
||||
t.Errorf("QueryProfiles(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.name, startTime, endTime, pType, err)
|
||||
continue
|
||||
}
|
||||
if err := pr.HasFunction("busywork"); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Automatically generated by MockGen. DO NOT EDIT!
|
||||
// Source: google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2 (interfaces: ProfilerServiceClient)
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
context "golang.org/x/net/context"
|
||||
v2 "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
|
||||
grpc "google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// Mock of ProfilerServiceClient interface
|
||||
type MockProfilerServiceClient struct {
|
||||
ctrl *gomock.Controller
|
||||
recorder *_MockProfilerServiceClientRecorder
|
||||
}
|
||||
|
||||
// Recorder for MockProfilerServiceClient (not exported)
|
||||
type _MockProfilerServiceClientRecorder struct {
|
||||
mock *MockProfilerServiceClient
|
||||
}
|
||||
|
||||
func NewMockProfilerServiceClient(ctrl *gomock.Controller) *MockProfilerServiceClient {
|
||||
mock := &MockProfilerServiceClient{ctrl: ctrl}
|
||||
mock.recorder = &_MockProfilerServiceClientRecorder{mock}
|
||||
return mock
|
||||
}
|
||||
|
||||
func (_m *MockProfilerServiceClient) EXPECT() *_MockProfilerServiceClientRecorder {
|
||||
return _m.recorder
|
||||
}
|
||||
|
||||
func (_m *MockProfilerServiceClient) CreateProfile(_param0 context.Context, _param1 *v2.CreateProfileRequest, _param2 ...grpc.CallOption) (*v2.Profile, error) {
|
||||
_s := []interface{}{_param0, _param1}
|
||||
for _, _x := range _param2 {
|
||||
_s = append(_s, _x)
|
||||
}
|
||||
ret := _m.ctrl.Call(_m, "CreateProfile", _s...)
|
||||
ret0, _ := ret[0].(*v2.Profile)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
func (_mr *_MockProfilerServiceClientRecorder) CreateProfile(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
|
||||
_s := append([]interface{}{arg0, arg1}, arg2...)
|
||||
return _mr.mock.ctrl.RecordCall(_mr.mock, "CreateProfile", _s...)
|
||||
}
|
||||
|
||||
func (_m *MockProfilerServiceClient) UpdateProfile(_param0 context.Context, _param1 *v2.UpdateProfileRequest, _param2 ...grpc.CallOption) (*v2.Profile, error) {
|
||||
_s := []interface{}{_param0, _param1}
|
||||
for _, _x := range _param2 {
|
||||
_s = append(_s, _x)
|
||||
}
|
||||
ret := _m.ctrl.Call(_m, "UpdateProfile", _s...)
|
||||
ret0, _ := ret[0].(*v2.Profile)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
func (_mr *_MockProfilerServiceClientRecorder) UpdateProfile(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
|
||||
_s := append([]interface{}{arg0, arg1}, arg2...)
|
||||
return _mr.mock.ctrl.RecordCall(_mr.mock, "UpdateProfile", _s...)
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build go1.8
|
||||
|
||||
package profiler
|
||||
|
||||
import "runtime"
|
||||
|
||||
func enableMutexProfiling() bool {
|
||||
// One percent of mutex contention events are profiled.
|
||||
runtime.SetMutexProfileFraction(100)
|
||||
return true
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !go1.8
|
||||
|
||||
package profiler
|
||||
|
||||
func enableMutexProfiling() bool {
|
||||
return false
|
||||
}
|
||||
+506
@@ -0,0 +1,506 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package profiler is a client for the Stackdriver Profiler service.
|
||||
//
|
||||
// This package is still experimental and subject to change.
|
||||
//
|
||||
// Usage example:
|
||||
//
|
||||
// import "cloud.google.com/go/profiler"
|
||||
// ...
|
||||
// if err := profiler.Start(profiler.Config{Service: "my-service"}); err != nil {
|
||||
// // TODO: Handle error.
|
||||
// }
|
||||
//
|
||||
// Calling Start will start a goroutine to collect profiles and upload to
|
||||
// the profiler server, at the rhythm specified by the server.
|
||||
//
|
||||
// The caller must provide the service string in the config, and may provide
|
||||
// other information as well. See Config for details.
|
||||
//
|
||||
// Profiler has CPU, heap and goroutine profiling enabled by default. Mutex
|
||||
// profiling can be enabled in the config. Note that goroutine and mutex
|
||||
// profiles are shown as "threads" and "contention" profiles in the profiler
|
||||
// UI.
|
||||
package profiler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
gcemd "cloud.google.com/go/compute/metadata"
|
||||
"cloud.google.com/go/internal/version"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
"github.com/google/pprof/profile"
|
||||
gax "github.com/googleapis/gax-go"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/option"
|
||||
gtransport "google.golang.org/api/transport/grpc"
|
||||
pb "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
|
||||
edpb "google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
grpcmd "google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
var (
|
||||
config Config
|
||||
startOnce sync.Once
|
||||
mutexEnabled bool
|
||||
// The functions below are stubbed to be overrideable for testing.
|
||||
getProjectID = gcemd.ProjectID
|
||||
getInstanceName = gcemd.InstanceName
|
||||
getZone = gcemd.Zone
|
||||
startCPUProfile = pprof.StartCPUProfile
|
||||
stopCPUProfile = pprof.StopCPUProfile
|
||||
writeHeapProfile = pprof.WriteHeapProfile
|
||||
sleep = gax.Sleep
|
||||
dialGRPC = gtransport.Dial
|
||||
onGCE = gcemd.OnGCE
|
||||
)
|
||||
|
||||
const (
|
||||
apiAddress = "cloudprofiler.googleapis.com:443"
|
||||
xGoogAPIMetadata = "x-goog-api-client"
|
||||
zoneNameLabel = "zone"
|
||||
versionLabel = "version"
|
||||
instanceLabel = "instance"
|
||||
scope = "https://www.googleapis.com/auth/monitoring.write"
|
||||
|
||||
initialBackoff = time.Second
|
||||
// Ensure the agent will recover within 1 hour.
|
||||
maxBackoff = time.Hour
|
||||
backoffMultiplier = 1.3 // Backoff envelope increases by this factor on each retry.
|
||||
retryInfoMetadata = "google.rpc.retryinfo-bin"
|
||||
)
|
||||
|
||||
// Config is the profiler configuration.
|
||||
type Config struct {
|
||||
// Service (or deprecated Target) must be provided to start the profiler.
|
||||
// It specifies the name of the service under which the profiled data
|
||||
// will be recorded and exposed at the Profiler UI for the project.
|
||||
// You can specify an arbitrary string, but see Deployment.target at
|
||||
// https://github.com/googleapis/googleapis/blob/master/google/devtools/cloudprofiler/v2/profiler.proto
|
||||
// for restrictions.
|
||||
// NOTE: The string should be the same across different replicas of
|
||||
// your service so that the globally constant profiling rate is
|
||||
// maintained. Do not put things like PID or unique pod ID in the name.
|
||||
Service string
|
||||
|
||||
// ServiceVersion is an optional field specifying the version of the
|
||||
// service. It can be an arbitrary string. Profiler profiles
|
||||
// once per minute for each version of each service in each zone.
|
||||
// ServiceVersion defaults to an empty string.
|
||||
ServiceVersion string
|
||||
|
||||
// DebugLogging enables detailed debug logging from profiler. It
|
||||
// defaults to false.
|
||||
DebugLogging bool
|
||||
|
||||
// MutexProfiling enables mutex profiling. It defaults to false.
|
||||
// Note that mutex profiling is not supported by Go versions older
|
||||
// than Go 1.8.
|
||||
MutexProfiling bool
|
||||
|
||||
// When true, collecting the heap profiles is disabled.
|
||||
NoHeapProfiling bool
|
||||
|
||||
// When true, collecting the goroutine profiles is disabled.
|
||||
NoGoroutineProfiling bool
|
||||
|
||||
// ProjectID is the Cloud Console project ID to use instead of
|
||||
// the one read from the VM metadata server.
|
||||
//
|
||||
// Set this if you are running the agent in your local environment
|
||||
// or anywhere else outside of Google Cloud Platform.
|
||||
ProjectID string
|
||||
|
||||
// APIAddr is the HTTP endpoint to use to connect to the profiler
|
||||
// agent API. Defaults to the production environment, overridable
|
||||
// for testing.
|
||||
APIAddr string
|
||||
|
||||
// Target is deprecated, use Service instead.
|
||||
Target string
|
||||
|
||||
instance string
|
||||
zone string
|
||||
}
|
||||
|
||||
// startError represents the error occured during the
|
||||
// initializating and starting of the agent.
|
||||
var startError error
|
||||
|
||||
// Start starts a goroutine to collect and upload profiles. The
|
||||
// caller must provide the service string in the config. See
|
||||
// Config for details. Start should only be called once. Any
|
||||
// additional calls will be ignored.
|
||||
func Start(cfg Config, options ...option.ClientOption) error {
|
||||
startOnce.Do(func() {
|
||||
startError = start(cfg, options...)
|
||||
})
|
||||
return startError
|
||||
}
|
||||
|
||||
func start(cfg Config, options ...option.ClientOption) error {
|
||||
if err := initializeConfig(cfg); err != nil {
|
||||
debugLog("failed to initialize config: %v", err)
|
||||
return err
|
||||
}
|
||||
if config.MutexProfiling {
|
||||
if mutexEnabled = enableMutexProfiling(); !mutexEnabled {
|
||||
return fmt.Errorf("mutex profiling is not supported by %s, requires Go 1.8 or later", runtime.Version())
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
opts := []option.ClientOption{
|
||||
option.WithEndpoint(config.APIAddr),
|
||||
option.WithScopes(scope),
|
||||
}
|
||||
opts = append(opts, options...)
|
||||
|
||||
conn, err := dialGRPC(ctx, opts...)
|
||||
if err != nil {
|
||||
debugLog("failed to dial GRPC: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
a := initializeAgent(pb.NewProfilerServiceClient(conn))
|
||||
go pollProfilerService(withXGoogHeader(ctx), a)
|
||||
return nil
|
||||
}
|
||||
|
||||
func debugLog(format string, e ...interface{}) {
|
||||
if config.DebugLogging {
|
||||
log.Printf(format, e...)
|
||||
}
|
||||
}
|
||||
|
||||
// agent polls the profiler server for instructions on behalf of a task,
|
||||
// and collects and uploads profiles as requested.
|
||||
type agent struct {
|
||||
client pb.ProfilerServiceClient
|
||||
deployment *pb.Deployment
|
||||
profileLabels map[string]string
|
||||
profileTypes []pb.ProfileType
|
||||
}
|
||||
|
||||
// abortedBackoffDuration retrieves the retry duration from gRPC trailing
|
||||
// metadata, which is set by the profiler server.
|
||||
func abortedBackoffDuration(md grpcmd.MD) (time.Duration, error) {
|
||||
elem := md[retryInfoMetadata]
|
||||
if len(elem) <= 0 {
|
||||
return 0, errors.New("no retry info")
|
||||
}
|
||||
|
||||
var retryInfo edpb.RetryInfo
|
||||
if err := proto.Unmarshal([]byte(elem[0]), &retryInfo); err != nil {
|
||||
return 0, err
|
||||
} else if time, err := ptypes.Duration(retryInfo.RetryDelay); err != nil {
|
||||
return 0, err
|
||||
} else {
|
||||
if time < 0 {
|
||||
return 0, errors.New("negative retry duration")
|
||||
}
|
||||
return time, nil
|
||||
}
|
||||
}
|
||||
|
||||
type retryer struct {
|
||||
backoff gax.Backoff
|
||||
md grpcmd.MD
|
||||
}
|
||||
|
||||
func (r *retryer) Retry(err error) (time.Duration, bool) {
|
||||
st, _ := status.FromError(err)
|
||||
if st != nil && st.Code() == codes.Aborted {
|
||||
dur, err := abortedBackoffDuration(r.md)
|
||||
if err == nil {
|
||||
return dur, true
|
||||
}
|
||||
debugLog("failed to get backoff duration: %v", err)
|
||||
}
|
||||
return r.backoff.Pause(), true
|
||||
}
|
||||
|
||||
// createProfile talks to the profiler server to create profile. In
|
||||
// case of error, the goroutine will sleep and retry. Sleep duration may
|
||||
// be specified by the server. Otherwise it will be an exponentially
|
||||
// increasing value, bounded by maxBackoff.
|
||||
func (a *agent) createProfile(ctx context.Context) *pb.Profile {
|
||||
req := pb.CreateProfileRequest{
|
||||
Deployment: a.deployment,
|
||||
ProfileType: a.profileTypes,
|
||||
}
|
||||
|
||||
var p *pb.Profile
|
||||
md := grpcmd.New(map[string]string{})
|
||||
|
||||
gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
|
||||
var err error
|
||||
p, err = a.client.CreateProfile(ctx, &req, grpc.Trailer(&md))
|
||||
if err != nil {
|
||||
debugLog("failed to create a profile, will retry: %v", err)
|
||||
}
|
||||
return err
|
||||
}, gax.WithRetry(func() gax.Retryer {
|
||||
return &retryer{
|
||||
backoff: gax.Backoff{
|
||||
Initial: initialBackoff,
|
||||
Max: maxBackoff,
|
||||
Multiplier: backoffMultiplier,
|
||||
},
|
||||
md: md,
|
||||
}
|
||||
}))
|
||||
|
||||
debugLog("successfully created profile %v", p.GetProfileType())
|
||||
return p
|
||||
}
|
||||
|
||||
func (a *agent) profileAndUpload(ctx context.Context, p *pb.Profile) {
|
||||
var prof bytes.Buffer
|
||||
pt := p.GetProfileType()
|
||||
|
||||
switch pt {
|
||||
case pb.ProfileType_CPU:
|
||||
duration, err := ptypes.Duration(p.Duration)
|
||||
if err != nil {
|
||||
debugLog("failed to get profile duration: %v", err)
|
||||
return
|
||||
}
|
||||
if err := startCPUProfile(&prof); err != nil {
|
||||
debugLog("failed to start CPU profile: %v", err)
|
||||
return
|
||||
}
|
||||
sleep(ctx, duration)
|
||||
stopCPUProfile()
|
||||
case pb.ProfileType_HEAP:
|
||||
if err := writeHeapProfile(&prof); err != nil {
|
||||
debugLog("failed to write heap profile: %v", err)
|
||||
return
|
||||
}
|
||||
case pb.ProfileType_THREADS:
|
||||
if err := pprof.Lookup("goroutine").WriteTo(&prof, 0); err != nil {
|
||||
debugLog("failed to create goroutine profile: %v", err)
|
||||
return
|
||||
}
|
||||
case pb.ProfileType_CONTENTION:
|
||||
duration, err := ptypes.Duration(p.Duration)
|
||||
if err != nil {
|
||||
debugLog("failed to get profile duration: %v", err)
|
||||
return
|
||||
}
|
||||
if err := deltaMutexProfile(ctx, duration, &prof); err != nil {
|
||||
debugLog("failed to create mutex profile: %v", err)
|
||||
return
|
||||
}
|
||||
default:
|
||||
debugLog("unexpected profile type: %v", pt)
|
||||
return
|
||||
}
|
||||
|
||||
// Starting Go 1.9 the profiles are symbolized by runtime/pprof.
|
||||
// TODO(jianqiaoli): Remove the symbolization code when we decide to
|
||||
// stop supporting Go 1.8.
|
||||
if !shouldAssumeSymbolized && pt != pb.ProfileType_CONTENTION {
|
||||
if err := parseAndSymbolize(&prof); err != nil {
|
||||
debugLog("failed to symbolize profile: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
p.ProfileBytes = prof.Bytes()
|
||||
p.Labels = a.profileLabels
|
||||
req := pb.UpdateProfileRequest{Profile: p}
|
||||
|
||||
// Upload profile, discard profile in case of error.
|
||||
debugLog("start uploading profile")
|
||||
if _, err := a.client.UpdateProfile(ctx, &req); err != nil {
|
||||
debugLog("failed to upload profile: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// deltaMutexProfile writes mutex profile changes over a time period specified
|
||||
// with 'duration' to 'prof'.
|
||||
func deltaMutexProfile(ctx context.Context, duration time.Duration, prof *bytes.Buffer) error {
|
||||
if !mutexEnabled {
|
||||
return errors.New("mutex profiling is not enabled")
|
||||
}
|
||||
p0, err := mutexProfile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sleep(ctx, duration)
|
||||
p, err := mutexProfile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO(jianqiaoli): Remove this check when github.com/google/pprof/issues/242
|
||||
// is fixed.
|
||||
if len(p0.Mapping) > 0 {
|
||||
p0.Scale(-1)
|
||||
p, err = profile.Merge([]*profile.Profile{p0, p})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// The mutex profile is not symbolized by runtime.pprof until
|
||||
// golang.org/issue/21474 is fixed in go1.10.
|
||||
symbolize(p)
|
||||
return p.Write(prof)
|
||||
}
|
||||
|
||||
func mutexProfile() (*profile.Profile, error) {
|
||||
p := pprof.Lookup("mutex")
|
||||
if p == nil {
|
||||
return nil, errors.New("mutex profiling is not supported")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := p.WriteTo(&buf, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return profile.Parse(&buf)
|
||||
}
|
||||
|
||||
// withXGoogHeader sets the name and version of the application in
|
||||
// the `x-goog-api-client` header passed on each request. Intended for
|
||||
// use by Google-written clients.
|
||||
func withXGoogHeader(ctx context.Context, keyval ...string) context.Context {
|
||||
kv := append([]string{"gl-go", version.Go(), "gccl", version.Repo}, keyval...)
|
||||
kv = append(kv, "gax", gax.Version, "grpc", grpc.Version)
|
||||
|
||||
md, _ := grpcmd.FromOutgoingContext(ctx)
|
||||
md = md.Copy()
|
||||
md[xGoogAPIMetadata] = []string{gax.XGoogHeader(kv...)}
|
||||
return grpcmd.NewOutgoingContext(ctx, md)
|
||||
}
|
||||
|
||||
func initializeAgent(c pb.ProfilerServiceClient) *agent {
|
||||
labels := map[string]string{}
|
||||
if config.zone != "" {
|
||||
labels[zoneNameLabel] = config.zone
|
||||
}
|
||||
if config.ServiceVersion != "" {
|
||||
labels[versionLabel] = config.ServiceVersion
|
||||
}
|
||||
d := &pb.Deployment{
|
||||
ProjectId: config.ProjectID,
|
||||
Target: config.Target,
|
||||
Labels: labels,
|
||||
}
|
||||
|
||||
profileLabels := map[string]string{}
|
||||
|
||||
if config.instance != "" {
|
||||
profileLabels[instanceLabel] = config.instance
|
||||
}
|
||||
|
||||
profileTypes := []pb.ProfileType{pb.ProfileType_CPU}
|
||||
if !config.NoHeapProfiling {
|
||||
profileTypes = append(profileTypes, pb.ProfileType_HEAP)
|
||||
}
|
||||
if !config.NoGoroutineProfiling {
|
||||
profileTypes = append(profileTypes, pb.ProfileType_THREADS)
|
||||
}
|
||||
if mutexEnabled {
|
||||
profileTypes = append(profileTypes, pb.ProfileType_CONTENTION)
|
||||
}
|
||||
|
||||
return &agent{
|
||||
client: c,
|
||||
deployment: d,
|
||||
profileLabels: profileLabels,
|
||||
profileTypes: profileTypes,
|
||||
}
|
||||
}
|
||||
|
||||
func initializeConfig(cfg Config) error {
|
||||
config = cfg
|
||||
|
||||
switch {
|
||||
case config.Service != "":
|
||||
config.Target = config.Service
|
||||
case config.Target == "":
|
||||
config.Target = os.Getenv("GAE_SERVICE")
|
||||
}
|
||||
|
||||
if config.Target == "" {
|
||||
return errors.New("service name must be specified in the configuration")
|
||||
}
|
||||
|
||||
if config.ServiceVersion == "" {
|
||||
config.ServiceVersion = os.Getenv("GAE_VERSION")
|
||||
}
|
||||
|
||||
if projectID := os.Getenv("GOOGLE_CLOUD_PROJECT"); config.ProjectID == "" && projectID != "" {
|
||||
// Cloud Shell and App Engine set this environment variable to the project
|
||||
// ID, so use it if present. In case of App Engine the project ID is also
|
||||
// available from the GCE metadata server, but by using the environment
|
||||
// variable saves one request to the metadata server. The environment
|
||||
// project ID is only used if no project ID is provided in the
|
||||
// configuration.
|
||||
config.ProjectID = projectID
|
||||
}
|
||||
if onGCE() {
|
||||
var err error
|
||||
if config.ProjectID == "" {
|
||||
if config.ProjectID, err = getProjectID(); err != nil {
|
||||
return fmt.Errorf("failed to get the project ID from Compute Engine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if config.zone, err = getZone(); err != nil {
|
||||
return fmt.Errorf("failed to get zone from Compute Engine: %v", err)
|
||||
}
|
||||
|
||||
if config.instance, err = getInstanceName(); err != nil {
|
||||
return fmt.Errorf("failed to get instance from Compute Engine: %v", err)
|
||||
}
|
||||
|
||||
} else {
|
||||
if config.ProjectID == "" {
|
||||
return fmt.Errorf("project ID must be specified in the configuration if running outside of GCP")
|
||||
}
|
||||
}
|
||||
|
||||
if config.APIAddr == "" {
|
||||
config.APIAddr = apiAddress
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pollProfilerService starts an endless loop to poll the profiler
|
||||
// server for instructions, and collects and uploads profiles as
|
||||
// requested.
|
||||
func pollProfilerService(ctx context.Context, a *agent) {
|
||||
debugLog("profiler has started")
|
||||
for {
|
||||
p := a.createProfile(ctx)
|
||||
a.profileAndUpload(ctx, p)
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package profiler_test
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/profiler"
|
||||
)
|
||||
|
||||
func ExampleStart() {
|
||||
if err := profiler.Start(profiler.Config{Service: "my-service", ServiceVersion: "v1"}); err != nil {
|
||||
//TODO: Handle error.
|
||||
}
|
||||
}
|
||||
+876
@@ -0,0 +1,876 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package profiler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/internal/testutil"
|
||||
"cloud.google.com/go/profiler/mocks"
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/golang/protobuf/ptypes"
|
||||
"github.com/google/pprof/profile"
|
||||
gax "github.com/googleapis/gax-go"
|
||||
"golang.org/x/net/context"
|
||||
gtransport "google.golang.org/api/transport/grpc"
|
||||
pb "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
|
||||
edpb "google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc/codes"
|
||||
grpcmd "google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
testProjectID = "test-project-ID"
|
||||
testInstance = "test-instance"
|
||||
testZone = "test-zone"
|
||||
testTarget = "test-target"
|
||||
testService = "test-service"
|
||||
testSvcVersion = "test-service-version"
|
||||
testProfileDuration = time.Second * 10
|
||||
testServerTimeout = time.Second * 15
|
||||
)
|
||||
|
||||
func createTestDeployment() *pb.Deployment {
|
||||
labels := map[string]string{
|
||||
zoneNameLabel: testZone,
|
||||
versionLabel: testSvcVersion,
|
||||
}
|
||||
return &pb.Deployment{
|
||||
ProjectId: testProjectID,
|
||||
Target: testService,
|
||||
Labels: labels,
|
||||
}
|
||||
}
|
||||
|
||||
func createTestAgent(psc pb.ProfilerServiceClient) *agent {
|
||||
return &agent{
|
||||
client: psc,
|
||||
deployment: createTestDeployment(),
|
||||
profileLabels: map[string]string{instanceLabel: testInstance},
|
||||
profileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
|
||||
}
|
||||
}
|
||||
|
||||
func createTrailers(dur time.Duration) map[string]string {
|
||||
b, _ := proto.Marshal(&edpb.RetryInfo{
|
||||
RetryDelay: ptypes.DurationProto(dur),
|
||||
})
|
||||
return map[string]string{
|
||||
retryInfoMetadata: string(b),
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateProfile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
mpc := mocks.NewMockProfilerServiceClient(ctrl)
|
||||
a := createTestAgent(mpc)
|
||||
p := &pb.Profile{Name: "test_profile"}
|
||||
wantRequest := pb.CreateProfileRequest{
|
||||
Deployment: a.deployment,
|
||||
ProfileType: a.profileTypes,
|
||||
}
|
||||
|
||||
mpc.EXPECT().CreateProfile(ctx, gomock.Eq(&wantRequest), gomock.Any()).Times(1).Return(p, nil)
|
||||
|
||||
gotP := a.createProfile(ctx)
|
||||
|
||||
if !testutil.Equal(gotP, p) {
|
||||
t.Errorf("CreateProfile() got wrong profile, got %v, want %v", gotP, p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfileAndUpload(t *testing.T) {
|
||||
oldStartCPUProfile, oldStopCPUProfile, oldWriteHeapProfile, oldSleep := startCPUProfile, stopCPUProfile, writeHeapProfile, sleep
|
||||
defer func() {
|
||||
startCPUProfile, stopCPUProfile, writeHeapProfile, sleep = oldStartCPUProfile, oldStopCPUProfile, oldWriteHeapProfile, oldSleep
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
|
||||
errFunc := func(io.Writer) error { return errors.New("") }
|
||||
testDuration := time.Second * 5
|
||||
tests := []struct {
|
||||
profileType pb.ProfileType
|
||||
duration *time.Duration
|
||||
startCPUProfileFunc func(io.Writer) error
|
||||
writeHeapProfileFunc func(io.Writer) error
|
||||
wantBytes []byte
|
||||
}{
|
||||
{
|
||||
profileType: pb.ProfileType_CPU,
|
||||
duration: &testDuration,
|
||||
startCPUProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{1})
|
||||
return nil
|
||||
},
|
||||
writeHeapProfileFunc: errFunc,
|
||||
wantBytes: []byte{1},
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_CPU,
|
||||
startCPUProfileFunc: errFunc,
|
||||
writeHeapProfileFunc: errFunc,
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_CPU,
|
||||
duration: &testDuration,
|
||||
startCPUProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{2})
|
||||
return nil
|
||||
},
|
||||
writeHeapProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{3})
|
||||
return nil
|
||||
},
|
||||
wantBytes: []byte{2},
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_HEAP,
|
||||
startCPUProfileFunc: errFunc,
|
||||
writeHeapProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{4})
|
||||
return nil
|
||||
},
|
||||
wantBytes: []byte{4},
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_HEAP,
|
||||
startCPUProfileFunc: errFunc,
|
||||
writeHeapProfileFunc: errFunc,
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_HEAP,
|
||||
startCPUProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{5})
|
||||
return nil
|
||||
},
|
||||
writeHeapProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{6})
|
||||
return nil
|
||||
},
|
||||
wantBytes: []byte{6},
|
||||
},
|
||||
{
|
||||
profileType: pb.ProfileType_PROFILE_TYPE_UNSPECIFIED,
|
||||
startCPUProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{7})
|
||||
return nil
|
||||
},
|
||||
writeHeapProfileFunc: func(w io.Writer) error {
|
||||
w.Write([]byte{8})
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
mpc := mocks.NewMockProfilerServiceClient(ctrl)
|
||||
a := createTestAgent(mpc)
|
||||
startCPUProfile = tt.startCPUProfileFunc
|
||||
stopCPUProfile = func() {}
|
||||
writeHeapProfile = tt.writeHeapProfileFunc
|
||||
var gotSleep *time.Duration
|
||||
sleep = func(ctx context.Context, d time.Duration) error {
|
||||
gotSleep = &d
|
||||
return nil
|
||||
}
|
||||
p := &pb.Profile{ProfileType: tt.profileType}
|
||||
if tt.duration != nil {
|
||||
p.Duration = ptypes.DurationProto(*tt.duration)
|
||||
}
|
||||
if tt.wantBytes != nil {
|
||||
wantProfile := &pb.Profile{
|
||||
ProfileType: p.ProfileType,
|
||||
Duration: p.Duration,
|
||||
ProfileBytes: tt.wantBytes,
|
||||
Labels: a.profileLabels,
|
||||
}
|
||||
wantRequest := pb.UpdateProfileRequest{
|
||||
Profile: wantProfile,
|
||||
}
|
||||
mpc.EXPECT().UpdateProfile(ctx, gomock.Eq(&wantRequest)).Times(1)
|
||||
} else {
|
||||
mpc.EXPECT().UpdateProfile(gomock.Any(), gomock.Any()).MaxTimes(0)
|
||||
}
|
||||
|
||||
a.profileAndUpload(ctx, p)
|
||||
|
||||
if tt.duration == nil {
|
||||
if gotSleep != nil {
|
||||
t.Errorf("profileAndUpload(%v) slept for: %v, want no sleep", p, gotSleep)
|
||||
}
|
||||
} else {
|
||||
if gotSleep == nil {
|
||||
t.Errorf("profileAndUpload(%v) didn't sleep, want sleep for: %v", p, tt.duration)
|
||||
} else if *gotSleep != *tt.duration {
|
||||
t.Errorf("profileAndUpload(%v) slept for wrong duration, got: %v, want: %v", p, gotSleep, tt.duration)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetry(t *testing.T) {
|
||||
normalDuration := time.Second * 3
|
||||
negativeDuration := time.Second * -3
|
||||
|
||||
tests := []struct {
|
||||
trailers map[string]string
|
||||
wantPause *time.Duration
|
||||
}{
|
||||
{
|
||||
createTrailers(normalDuration),
|
||||
&normalDuration,
|
||||
},
|
||||
{
|
||||
createTrailers(negativeDuration),
|
||||
nil,
|
||||
},
|
||||
{
|
||||
map[string]string{retryInfoMetadata: "wrong format"},
|
||||
nil,
|
||||
},
|
||||
{
|
||||
map[string]string{},
|
||||
nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
md := grpcmd.New(tt.trailers)
|
||||
r := &retryer{
|
||||
backoff: gax.Backoff{
|
||||
Initial: initialBackoff,
|
||||
Max: maxBackoff,
|
||||
Multiplier: backoffMultiplier,
|
||||
},
|
||||
md: md,
|
||||
}
|
||||
|
||||
pause, shouldRetry := r.Retry(status.Error(codes.Aborted, ""))
|
||||
|
||||
if !shouldRetry {
|
||||
t.Error("retryer.Retry() returned shouldRetry false, want true")
|
||||
}
|
||||
|
||||
if tt.wantPause != nil {
|
||||
if pause != *tt.wantPause {
|
||||
t.Errorf("retryer.Retry() returned wrong pause, got: %v, want: %v", pause, tt.wantPause)
|
||||
}
|
||||
} else {
|
||||
if pause > initialBackoff {
|
||||
t.Errorf("retryer.Retry() returned wrong pause, got: %v, want: < %v", pause, initialBackoff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
md := grpcmd.New(map[string]string{})
|
||||
|
||||
r := &retryer{
|
||||
backoff: gax.Backoff{
|
||||
Initial: initialBackoff,
|
||||
Max: maxBackoff,
|
||||
Multiplier: backoffMultiplier,
|
||||
},
|
||||
md: md,
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
pause, shouldRetry := r.Retry(errors.New(""))
|
||||
if !shouldRetry {
|
||||
t.Errorf("retryer.Retry() called %v times, returned shouldRetry false, want true", i)
|
||||
}
|
||||
if pause > maxBackoff {
|
||||
t.Errorf("retryer.Retry() called %v times, returned wrong pause, got: %v, want: < %v", i, pause, maxBackoff)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithXGoogHeader(t *testing.T) {
|
||||
ctx := withXGoogHeader(context.Background())
|
||||
md, _ := grpcmd.FromOutgoingContext(ctx)
|
||||
|
||||
if xg := md[xGoogAPIMetadata]; len(xg) == 0 {
|
||||
t.Errorf("withXGoogHeader() sets empty xGoogHeader")
|
||||
} else {
|
||||
if !strings.Contains(xg[0], "gl-go/") {
|
||||
t.Errorf("withXGoogHeader() got: %v, want gl-go key", xg[0])
|
||||
}
|
||||
if !strings.Contains(xg[0], "gccl/") {
|
||||
t.Errorf("withXGoogHeader() got: %v, want gccl key", xg[0])
|
||||
}
|
||||
if !strings.Contains(xg[0], "gax/") {
|
||||
t.Errorf("withXGoogHeader() got: %v, want gax key", xg[0])
|
||||
}
|
||||
if !strings.Contains(xg[0], "grpc/") {
|
||||
t.Errorf("withXGoogHeader() got: %v, want grpc key", xg[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeAgent(t *testing.T) {
|
||||
oldConfig, oldMutexEnabled := config, mutexEnabled
|
||||
defer func() {
|
||||
config, mutexEnabled = oldConfig, oldMutexEnabled
|
||||
}()
|
||||
|
||||
for _, tt := range []struct {
|
||||
config Config
|
||||
enableMutex bool
|
||||
wantProfileTypes []pb.ProfileType
|
||||
wantDeploymentLabels map[string]string
|
||||
wantProfileLabels map[string]string
|
||||
}{
|
||||
{
|
||||
config: Config{ServiceVersion: testSvcVersion, zone: testZone},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
|
||||
wantDeploymentLabels: map[string]string{zoneNameLabel: testZone, versionLabel: testSvcVersion},
|
||||
wantProfileLabels: map[string]string{},
|
||||
},
|
||||
{
|
||||
config: Config{zone: testZone},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
|
||||
wantDeploymentLabels: map[string]string{zoneNameLabel: testZone},
|
||||
wantProfileLabels: map[string]string{},
|
||||
},
|
||||
{
|
||||
config: Config{ServiceVersion: testSvcVersion},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
|
||||
wantDeploymentLabels: map[string]string{versionLabel: testSvcVersion},
|
||||
wantProfileLabels: map[string]string{},
|
||||
},
|
||||
{
|
||||
config: Config{instance: testInstance},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
|
||||
wantDeploymentLabels: map[string]string{},
|
||||
wantProfileLabels: map[string]string{instanceLabel: testInstance},
|
||||
},
|
||||
{
|
||||
config: Config{instance: testInstance},
|
||||
enableMutex: true,
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS, pb.ProfileType_CONTENTION},
|
||||
wantDeploymentLabels: map[string]string{},
|
||||
wantProfileLabels: map[string]string{instanceLabel: testInstance},
|
||||
},
|
||||
{
|
||||
config: Config{NoHeapProfiling: true},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_THREADS},
|
||||
wantDeploymentLabels: map[string]string{},
|
||||
wantProfileLabels: map[string]string{},
|
||||
},
|
||||
{
|
||||
config: Config{NoHeapProfiling: true, NoGoroutineProfiling: true},
|
||||
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU},
|
||||
wantDeploymentLabels: map[string]string{},
|
||||
wantProfileLabels: map[string]string{},
|
||||
},
|
||||
} {
|
||||
|
||||
config = tt.config
|
||||
config.ProjectID = testProjectID
|
||||
config.Target = testTarget
|
||||
mutexEnabled = tt.enableMutex
|
||||
a := initializeAgent(nil)
|
||||
|
||||
wantDeployment := &pb.Deployment{
|
||||
ProjectId: testProjectID,
|
||||
Target: testTarget,
|
||||
Labels: tt.wantDeploymentLabels,
|
||||
}
|
||||
if !testutil.Equal(a.deployment, wantDeployment) {
|
||||
t.Errorf("initializeAgent() got deployment: %v, want %v", a.deployment, wantDeployment)
|
||||
}
|
||||
if !testutil.Equal(a.profileLabels, tt.wantProfileLabels) {
|
||||
t.Errorf("initializeAgent() got profile labels: %v, want %v", a.profileLabels, tt.wantProfileLabels)
|
||||
}
|
||||
if !testutil.Equal(a.profileTypes, tt.wantProfileTypes) {
|
||||
t.Errorf("initializeAgent() got profile types: %v, want %v", a.profileTypes, tt.wantProfileTypes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitializeConfig(t *testing.T) {
|
||||
oldConfig, oldService, oldVersion, oldEnvProjectID, oldGetProjectID, oldGetInstanceName, oldGetZone, oldOnGCE := config, os.Getenv("GAE_SERVICE"), os.Getenv("GAE_VERSION"), os.Getenv("GOOGLE_CLOUD_PROJECT"), getProjectID, getInstanceName, getZone, onGCE
|
||||
defer func() {
|
||||
config, getProjectID, getInstanceName, getZone, onGCE = oldConfig, oldGetProjectID, oldGetInstanceName, oldGetZone, oldOnGCE
|
||||
if err := os.Setenv("GAE_SERVICE", oldService); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Setenv("GAE_VERSION", oldVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Setenv("GOOGLE_CLOUD_PROJECT", oldEnvProjectID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
const (
|
||||
testGAEService = "test-gae-service"
|
||||
testGAEVersion = "test-gae-version"
|
||||
testGCEProjectID = "test-gce-project-id"
|
||||
testEnvProjectID = "test-env-project-id"
|
||||
)
|
||||
for _, tt := range []struct {
|
||||
desc string
|
||||
config Config
|
||||
wantConfig Config
|
||||
wantErrorString string
|
||||
onGAE bool
|
||||
onGCE bool
|
||||
envProjectID bool
|
||||
}{
|
||||
{
|
||||
"accepts service name",
|
||||
Config{Service: testService},
|
||||
Config{Target: testService, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"accepts target name",
|
||||
Config{Target: testTarget},
|
||||
Config{Target: testTarget, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"env project overrides GCE project",
|
||||
Config{Service: testService},
|
||||
Config{Target: testService, ProjectID: testEnvProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"requires service name",
|
||||
Config{},
|
||||
Config{},
|
||||
"service name must be specified in the configuration",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"accepts service name from config and service version from GAE",
|
||||
Config{Service: testService},
|
||||
Config{Target: testService, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"accepts target name from config and service version from GAE",
|
||||
Config{Target: testTarget},
|
||||
Config{Target: testTarget, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"reads both service name and version from GAE env vars",
|
||||
Config{},
|
||||
Config{Target: testGAEService, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"accepts service version from config",
|
||||
Config{Service: testService, ServiceVersion: testSvcVersion},
|
||||
Config{Target: testService, ServiceVersion: testSvcVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"configured version has priority over GAE-provided version",
|
||||
Config{Service: testService, ServiceVersion: testSvcVersion},
|
||||
Config{Target: testService, ServiceVersion: testSvcVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"configured project ID has priority over metadata-provided project ID",
|
||||
Config{Service: testService, ProjectID: testProjectID},
|
||||
Config{Target: testService, ProjectID: testProjectID, zone: testZone, instance: testInstance},
|
||||
"",
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"configured project ID has priority over environment project ID",
|
||||
Config{Service: testService, ProjectID: testProjectID},
|
||||
Config{Target: testService, ProjectID: testProjectID},
|
||||
"",
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"requires project ID if not on GCE",
|
||||
Config{Service: testService},
|
||||
Config{Target: testService},
|
||||
"project ID must be specified in the configuration if running outside of GCP",
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
},
|
||||
} {
|
||||
t.Logf("Running test: %s", tt.desc)
|
||||
envService, envVersion := "", ""
|
||||
if tt.onGAE {
|
||||
envService, envVersion = testGAEService, testGAEVersion
|
||||
}
|
||||
if err := os.Setenv("GAE_SERVICE", envService); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Setenv("GAE_VERSION", envVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tt.onGCE {
|
||||
onGCE = func() bool { return true }
|
||||
getProjectID = func() (string, error) { return testGCEProjectID, nil }
|
||||
getZone = func() (string, error) { return testZone, nil }
|
||||
getInstanceName = func() (string, error) { return testInstance, nil }
|
||||
} else {
|
||||
onGCE = func() bool { return false }
|
||||
getProjectID = func() (string, error) { return "", fmt.Errorf("test get project id error") }
|
||||
getZone = func() (string, error) { return "", fmt.Errorf("test get zone error") }
|
||||
getInstanceName = func() (string, error) { return "", fmt.Errorf("test get instance error") }
|
||||
}
|
||||
envProjectID := ""
|
||||
if tt.envProjectID {
|
||||
envProjectID = testEnvProjectID
|
||||
}
|
||||
if err := os.Setenv("GOOGLE_CLOUD_PROJECT", envProjectID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errorString := ""
|
||||
if err := initializeConfig(tt.config); err != nil {
|
||||
errorString = err.Error()
|
||||
}
|
||||
|
||||
if !strings.Contains(errorString, tt.wantErrorString) {
|
||||
t.Errorf("initializeConfig(%v) got error: %v, want contain %v", tt.config, errorString, tt.wantErrorString)
|
||||
}
|
||||
if tt.wantErrorString == "" {
|
||||
tt.wantConfig.APIAddr = apiAddress
|
||||
}
|
||||
tt.wantConfig.Service = tt.config.Service
|
||||
if config != tt.wantConfig {
|
||||
t.Errorf("initializeConfig(%v) got: %v, want %v", tt.config, config, tt.wantConfig)
|
||||
}
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
wantErrorString string
|
||||
getProjectIDError bool
|
||||
getZoneError bool
|
||||
getInstanceError bool
|
||||
}{
|
||||
{
|
||||
wantErrorString: "failed to get the project ID from Compute Engine:",
|
||||
getProjectIDError: true,
|
||||
},
|
||||
{
|
||||
wantErrorString: "failed to get zone from Compute Engine:",
|
||||
getZoneError: true,
|
||||
},
|
||||
{
|
||||
wantErrorString: "failed to get instance from Compute Engine:",
|
||||
getInstanceError: true,
|
||||
},
|
||||
} {
|
||||
onGCE = func() bool { return true }
|
||||
if tt.getProjectIDError {
|
||||
getProjectID = func() (string, error) { return "", fmt.Errorf("test get project ID error") }
|
||||
} else {
|
||||
getProjectID = func() (string, error) { return testGCEProjectID, nil }
|
||||
}
|
||||
|
||||
if tt.getZoneError {
|
||||
getZone = func() (string, error) { return "", fmt.Errorf("test get zone error") }
|
||||
} else {
|
||||
getZone = func() (string, error) { return testZone, nil }
|
||||
}
|
||||
|
||||
if tt.getInstanceError {
|
||||
getInstanceName = func() (string, error) { return "", fmt.Errorf("test get instance error") }
|
||||
} else {
|
||||
getInstanceName = func() (string, error) { return testInstance, nil }
|
||||
}
|
||||
errorString := ""
|
||||
if err := initializeConfig(Config{Service: testService}); err != nil {
|
||||
errorString = err.Error()
|
||||
}
|
||||
|
||||
if !strings.Contains(errorString, tt.wantErrorString) {
|
||||
t.Errorf("initializeConfig() got error: %v, want contain %v", errorString, tt.wantErrorString)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeProfilerServer struct {
|
||||
pb.ProfilerServiceServer
|
||||
count int
|
||||
gotProfiles map[string][]byte
|
||||
done chan bool
|
||||
}
|
||||
|
||||
func (fs *fakeProfilerServer) CreateProfile(ctx context.Context, in *pb.CreateProfileRequest) (*pb.Profile, error) {
|
||||
fs.count++
|
||||
switch fs.count {
|
||||
case 1:
|
||||
return &pb.Profile{Name: "testCPU", ProfileType: pb.ProfileType_CPU, Duration: ptypes.DurationProto(testProfileDuration)}, nil
|
||||
case 2:
|
||||
return &pb.Profile{Name: "testHeap", ProfileType: pb.ProfileType_HEAP}, nil
|
||||
default:
|
||||
select {}
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *fakeProfilerServer) UpdateProfile(ctx context.Context, in *pb.UpdateProfileRequest) (*pb.Profile, error) {
|
||||
switch in.Profile.ProfileType {
|
||||
case pb.ProfileType_CPU:
|
||||
fs.gotProfiles["CPU"] = in.Profile.ProfileBytes
|
||||
case pb.ProfileType_HEAP:
|
||||
fs.gotProfiles["HEAP"] = in.Profile.ProfileBytes
|
||||
fs.done <- true
|
||||
}
|
||||
|
||||
return in.Profile, nil
|
||||
}
|
||||
|
||||
func profileeLoop(quit chan bool) {
|
||||
for {
|
||||
select {
|
||||
case <-quit:
|
||||
return
|
||||
default:
|
||||
profileeWork()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func profileeWork() {
|
||||
data := make([]byte, 1024*1024)
|
||||
rand.Read(data)
|
||||
|
||||
var b bytes.Buffer
|
||||
gz := gzip.NewWriter(&b)
|
||||
if _, err := gz.Write(data); err != nil {
|
||||
log.Println("failed to write to gzip stream", err)
|
||||
return
|
||||
}
|
||||
if err := gz.Flush(); err != nil {
|
||||
log.Println("failed to flush to gzip stream", err)
|
||||
return
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
log.Println("failed to close gzip stream", err)
|
||||
}
|
||||
}
|
||||
|
||||
func validateProfile(rawData []byte, wantFunctionName string) error {
|
||||
p, err := profile.ParseData(rawData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ParseData failed: %v", err)
|
||||
}
|
||||
|
||||
if len(p.Sample) == 0 {
|
||||
return fmt.Errorf("profile contains zero samples: %v", p)
|
||||
}
|
||||
|
||||
if len(p.Location) == 0 {
|
||||
return fmt.Errorf("profile contains zero locations: %v", p)
|
||||
}
|
||||
|
||||
if len(p.Function) == 0 {
|
||||
return fmt.Errorf("profile contains zero functions: %v", p)
|
||||
}
|
||||
|
||||
for _, l := range p.Location {
|
||||
if len(l.Line) > 0 && l.Line[0].Function != nil && strings.Contains(l.Line[0].Function.Name, wantFunctionName) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("wanted function name %s not found in the profile", wantFunctionName)
|
||||
}
|
||||
|
||||
func TestDeltaMutexProfile(t *testing.T) {
|
||||
oldMutexEnabled, oldMaxProcs := mutexEnabled, runtime.GOMAXPROCS(10)
|
||||
defer func() {
|
||||
mutexEnabled = oldMutexEnabled
|
||||
runtime.GOMAXPROCS(oldMaxProcs)
|
||||
}()
|
||||
if mutexEnabled = enableMutexProfiling(); !mutexEnabled {
|
||||
t.Skip("Go too old - mutex profiling not supported.")
|
||||
}
|
||||
|
||||
hog(time.Second, mutexHog)
|
||||
go func() {
|
||||
hog(2*time.Second, backgroundHog)
|
||||
}()
|
||||
|
||||
var prof bytes.Buffer
|
||||
if err := deltaMutexProfile(context.Background(), time.Second, &prof); err != nil {
|
||||
t.Fatalf("deltaMutexProfile() got error: %v", err)
|
||||
}
|
||||
p, err := profile.Parse(&prof)
|
||||
if err != nil {
|
||||
t.Fatalf("profile.Parse() got error: %v", err)
|
||||
}
|
||||
|
||||
if s := sum(p, "mutexHog"); s != 0 {
|
||||
t.Errorf("mutexHog found in the delta mutex profile (sum=%d):\n%s", s, p)
|
||||
}
|
||||
if s := sum(p, "backgroundHog"); s <= 0 {
|
||||
t.Errorf("backgroundHog not in the delta mutex profile (sum=%d):\n%s", s, p)
|
||||
}
|
||||
}
|
||||
|
||||
// sum returns the sum of all mutex counts from the samples whose
|
||||
// stacks include the specified function name.
|
||||
func sum(p *profile.Profile, fname string) int64 {
|
||||
locIDs := map[*profile.Location]bool{}
|
||||
for _, loc := range p.Location {
|
||||
for _, l := range loc.Line {
|
||||
if strings.Contains(l.Function.Name, fname) {
|
||||
locIDs[loc] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
var s int64
|
||||
for _, sample := range p.Sample {
|
||||
for _, loc := range sample.Location {
|
||||
if locIDs[loc] {
|
||||
s += sample.Value[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func mutexHog(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration) {
|
||||
for time.Since(start) < dt {
|
||||
mu1.Lock()
|
||||
runtime.Gosched()
|
||||
mu2.Lock()
|
||||
mu1.Unlock()
|
||||
mu2.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// backgroundHog is identical to mutexHog. We keep them separate
|
||||
// in order to distinguish them with function names in the stack trace.
|
||||
func backgroundHog(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration) {
|
||||
for time.Since(start) < dt {
|
||||
mu1.Lock()
|
||||
runtime.Gosched()
|
||||
mu2.Lock()
|
||||
mu1.Unlock()
|
||||
mu2.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func hog(dt time.Duration, hogger func(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration)) {
|
||||
start := time.Now()
|
||||
mu1 := new(sync.Mutex)
|
||||
mu2 := new(sync.Mutex)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(10)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
hogger(mu1, mu2, start, dt)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestAgentWithServer(t *testing.T) {
|
||||
oldDialGRPC, oldConfig := dialGRPC, config
|
||||
defer func() {
|
||||
dialGRPC, config = oldDialGRPC, oldConfig
|
||||
}()
|
||||
|
||||
srv, err := testutil.NewServer()
|
||||
if err != nil {
|
||||
t.Fatalf("testutil.NewServer(): %v", err)
|
||||
}
|
||||
fakeServer := &fakeProfilerServer{gotProfiles: map[string][]byte{}, done: make(chan bool)}
|
||||
pb.RegisterProfilerServiceServer(srv.Gsrv, fakeServer)
|
||||
|
||||
srv.Start()
|
||||
|
||||
dialGRPC = gtransport.DialInsecure
|
||||
if err := Start(Config{
|
||||
Target: testTarget,
|
||||
ProjectID: testProjectID,
|
||||
APIAddr: srv.Addr,
|
||||
instance: testInstance,
|
||||
zone: testZone,
|
||||
}); err != nil {
|
||||
t.Fatalf("Start(): %v", err)
|
||||
}
|
||||
|
||||
quitProfilee := make(chan bool)
|
||||
go profileeLoop(quitProfilee)
|
||||
|
||||
select {
|
||||
case <-fakeServer.done:
|
||||
case <-time.After(testServerTimeout):
|
||||
t.Errorf("got timeout after %v, want fake server done", testServerTimeout)
|
||||
}
|
||||
quitProfilee <- true
|
||||
|
||||
for _, pType := range []string{"CPU", "HEAP"} {
|
||||
if profile, ok := fakeServer.gotProfiles[pType]; !ok {
|
||||
t.Errorf("fakeServer.gotProfiles[%s] got no profile, want profile", pType)
|
||||
} else if err := validateProfile(profile, "profilee"); err != nil {
|
||||
t.Errorf("validateProfile(%s) got error: %v", pType, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+501
@@ -0,0 +1,501 @@
|
||||
// Copyright 2018 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Package proftest contains test helpers for profiler agent integration tests.
|
||||
// This package is experimental.
|
||||
|
||||
// golang.org/x/build/kubernetes/dialer.go imports "context" package (rather
|
||||
// than "golang.org/x/net/context") and that does not exist in Go 1.6 or
|
||||
// earlier.
|
||||
// +build go1.7
|
||||
|
||||
package proftest
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"golang.org/x/build/kubernetes"
|
||||
k8sapi "golang.org/x/build/kubernetes/api"
|
||||
"golang.org/x/build/kubernetes/gke"
|
||||
"golang.org/x/net/context"
|
||||
cloudbuild "google.golang.org/api/cloudbuild/v1"
|
||||
compute "google.golang.org/api/compute/v1"
|
||||
container "google.golang.org/api/container/v1"
|
||||
"google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
const (
|
||||
monitorWriteScope = "https://www.googleapis.com/auth/monitoring.write"
|
||||
storageReadScope = "https://www.googleapis.com/auth/devstorage.read_only"
|
||||
)
|
||||
|
||||
// TestRunner has common elements used for testing profiling agents on a range
|
||||
// of environments.
|
||||
type TestRunner struct {
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
// GCETestRunner supports testing a profiling agent on GCE.
|
||||
type GCETestRunner struct {
|
||||
TestRunner
|
||||
ComputeService *compute.Service
|
||||
}
|
||||
|
||||
// GKETestRunner supports testing a profiling agent on GKE.
|
||||
type GKETestRunner struct {
|
||||
TestRunner
|
||||
ContainerService *container.Service
|
||||
StorageClient *storage.Client
|
||||
Dockerfile string
|
||||
}
|
||||
|
||||
// ProfileResponse contains the response produced when querying profile server.
|
||||
type ProfileResponse struct {
|
||||
Profile ProfileData `json:"profile"`
|
||||
NumProfiles int32 `json:"numProfiles"`
|
||||
Deployments []interface{} `json:"deployments"`
|
||||
}
|
||||
|
||||
// ProfileData has data of a single profile.
|
||||
type ProfileData struct {
|
||||
Samples []int32 `json:"samples"`
|
||||
SampleMetrics interface{} `json:"sampleMetrics"`
|
||||
DefaultMetricType string `json:"defaultMetricType"`
|
||||
TreeNodes interface{} `json:"treeNodes"`
|
||||
Functions functionArray `json:"functions"`
|
||||
SourceFiles interface{} `json:"sourceFiles"`
|
||||
}
|
||||
|
||||
type functionArray struct {
|
||||
Name []string `json:"name"`
|
||||
Sourcefile []int32 `json:"sourceFile"`
|
||||
}
|
||||
|
||||
// InstanceConfig is configuration for starting single GCE instance for
|
||||
// profiling agent test case.
|
||||
type InstanceConfig struct {
|
||||
ProjectID string
|
||||
Zone string
|
||||
Name string
|
||||
StartupScript string
|
||||
MachineType string
|
||||
}
|
||||
|
||||
// ClusterConfig is configuration for starting single GKE cluster for profiling
|
||||
// agent test case.
|
||||
type ClusterConfig struct {
|
||||
ProjectID string
|
||||
Zone string
|
||||
ClusterName string
|
||||
PodName string
|
||||
ImageSourceName string
|
||||
ImageName string
|
||||
Bucket string
|
||||
Dockerfile string
|
||||
}
|
||||
|
||||
// HasFunction returns nil if the function is present, or, if the function is
|
||||
// not present, and error providing more details why the function is not
|
||||
// present.
|
||||
func (pr *ProfileResponse) HasFunction(functionName string) error {
|
||||
if pr.NumProfiles == 0 {
|
||||
return fmt.Errorf("failed to find function name %s in profile: profile response contains zero profiles: %v", functionName, pr)
|
||||
}
|
||||
if len(pr.Deployments) == 0 {
|
||||
return fmt.Errorf("failed to find function name %s in profile: profile response contains zero deployments: %v", functionName, pr)
|
||||
}
|
||||
if len(pr.Profile.Functions.Name) == 0 {
|
||||
return fmt.Errorf("failed to find function name %s in profile: profile does not have function data", functionName)
|
||||
}
|
||||
|
||||
for _, name := range pr.Profile.Functions.Name {
|
||||
if strings.Contains(name, functionName) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("failed to find function name %s in profile", functionName)
|
||||
}
|
||||
|
||||
// StartInstance starts a GCE Instance with name, zone, and projectId specified
|
||||
// by the inst, and which runs the startup script specified in inst.
|
||||
func (tr *GCETestRunner) StartInstance(ctx context.Context, inst *InstanceConfig) error {
|
||||
img, err := tr.ComputeService.Images.GetFromFamily("debian-cloud", "debian-9").Context(ctx).Do()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tr.ComputeService.Instances.Insert(inst.ProjectID, inst.Zone, &compute.Instance{
|
||||
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", inst.Zone, inst.MachineType),
|
||||
Name: inst.Name,
|
||||
Disks: []*compute.AttachedDisk{{
|
||||
AutoDelete: true, // delete the disk when the VM is deleted.
|
||||
Boot: true,
|
||||
Type: "PERSISTENT",
|
||||
Mode: "READ_WRITE",
|
||||
InitializeParams: &compute.AttachedDiskInitializeParams{
|
||||
SourceImage: img.SelfLink,
|
||||
DiskType: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/pd-standard", inst.ProjectID, inst.Zone),
|
||||
},
|
||||
}},
|
||||
NetworkInterfaces: []*compute.NetworkInterface{{
|
||||
Network: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/default", inst.ProjectID),
|
||||
AccessConfigs: []*compute.AccessConfig{{
|
||||
Name: "External NAT",
|
||||
}},
|
||||
}},
|
||||
Metadata: &compute.Metadata{
|
||||
Items: []*compute.MetadataItems{{
|
||||
Key: "startup-script",
|
||||
Value: googleapi.String(inst.StartupScript),
|
||||
}},
|
||||
},
|
||||
ServiceAccounts: []*compute.ServiceAccount{{
|
||||
Email: "default",
|
||||
Scopes: []string{
|
||||
monitorWriteScope,
|
||||
},
|
||||
}},
|
||||
}).Do()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteInstance deletes an instance with project id, name, and zone matched
|
||||
// by inst.
|
||||
func (tr *GCETestRunner) DeleteInstance(ctx context.Context, inst *InstanceConfig) error {
|
||||
if _, err := tr.ComputeService.Instances.Delete(inst.ProjectID, inst.Zone, inst.Name).Context(ctx).Do(); err != nil {
|
||||
return fmt.Errorf("Instances.Delete(%s) got error: %v", inst.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PollForSerialOutput polls serial port 2 of the GCE instance specified by
|
||||
// inst and returns when the finishString appears in the serial output
|
||||
// of the instance, or when the context times out.
|
||||
func (tr *GCETestRunner) PollForSerialOutput(ctx context.Context, inst *InstanceConfig, finishString string) error {
|
||||
var output string
|
||||
defer func() {
|
||||
log.Printf("Serial port output for %s:\n%s", inst.Name, output)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(20 * time.Second):
|
||||
resp, err := tr.ComputeService.Instances.GetSerialPortOutput(inst.ProjectID, inst.Zone, inst.Name).Port(2).Context(ctx).Do()
|
||||
if err != nil {
|
||||
// Transient failure.
|
||||
log.Printf("Transient error getting serial port output from instance %s (will retry): %v", inst.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if output = resp.Contents; strings.Contains(output, finishString) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// QueryProfiles retrieves profiles of a specific type, from a specific time
|
||||
// range, associated with a particular service and project.
|
||||
func (tr *TestRunner) QueryProfiles(projectID, service, startTime, endTime, profileType string) (ProfileResponse, error) {
|
||||
queryURL := fmt.Sprintf("https://cloudprofiler.googleapis.com/v2/projects/%s/profiles:query", projectID)
|
||||
const queryJSONFmt = `{"endTime": "%s", "profileType": "%s","startTime": "%s", "target": "%s"}`
|
||||
|
||||
queryRequest := fmt.Sprintf(queryJSONFmt, endTime, profileType, startTime, service)
|
||||
|
||||
resp, err := tr.Client.Post(queryURL, "application/json", strings.NewReader(queryRequest))
|
||||
if err != nil {
|
||||
return ProfileResponse{}, fmt.Errorf("failed to query API: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return ProfileResponse{}, fmt.Errorf("failed to read response body: %v", err)
|
||||
}
|
||||
|
||||
var pr ProfileResponse
|
||||
if err := json.Unmarshal(body, &pr); err != nil {
|
||||
return ProfileResponse{}, err
|
||||
}
|
||||
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
// createAndPublishDockerImage creates a docker image from source code in a GCS
|
||||
// bucket and pushes the image to Google Container Registry.
|
||||
func (tr *GKETestRunner) createAndPublishDockerImage(ctx context.Context, projectID, sourceBucket, sourceObject, ImageName string) error {
|
||||
cloudbuildService, err := cloudbuild.New(tr.Client)
|
||||
|
||||
build := &cloudbuild.Build{
|
||||
Source: &cloudbuild.Source{
|
||||
StorageSource: &cloudbuild.StorageSource{
|
||||
Bucket: sourceBucket,
|
||||
Object: sourceObject,
|
||||
},
|
||||
},
|
||||
Steps: []*cloudbuild.BuildStep{
|
||||
{
|
||||
Name: "gcr.io/cloud-builders/docker",
|
||||
Args: []string{"build", "-t", ImageName, "."},
|
||||
},
|
||||
},
|
||||
Images: []string{ImageName},
|
||||
}
|
||||
|
||||
op, err := cloudbuildService.Projects.Builds.Create(projectID, build).Context(ctx).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create image: %v", err)
|
||||
}
|
||||
opID := op.Name
|
||||
|
||||
// Wait for creating image.
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timed out waiting creating image")
|
||||
|
||||
case <-time.After(10 * time.Second):
|
||||
op, err := cloudbuildService.Operations.Get(opID).Context(ctx).Do()
|
||||
if err != nil {
|
||||
log.Printf("Transient error getting operation (will retry): %v", err)
|
||||
break
|
||||
}
|
||||
if op.Done == true {
|
||||
log.Printf("Published image %s to Google Container Registry.", ImageName)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type imageResponse struct {
|
||||
Manifest map[string]interface{} `json:"manifest"`
|
||||
Name string `json:"name"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
// deleteDockerImage deletes a docker image from Google Container Registry.
|
||||
func (tr *GKETestRunner) deleteDockerImage(ctx context.Context, ImageName string) []error {
|
||||
queryImageURL := fmt.Sprintf("https://gcr.io/v2/%s/tags/list", ImageName)
|
||||
resp, err := tr.Client.Get(queryImageURL)
|
||||
if err != nil {
|
||||
return []error{fmt.Errorf("failed to list tags: %v", err)}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
var ir imageResponse
|
||||
if err := json.Unmarshal(body, &ir); err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
|
||||
const deleteImageURLFmt = "https://gcr.io/v2/%s/manifests/%s"
|
||||
var errs []error
|
||||
for _, tag := range ir.Tags {
|
||||
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, tag)); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to delete tag %s: %v", tag, err))
|
||||
}
|
||||
}
|
||||
|
||||
for manifest := range ir.Manifest {
|
||||
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, manifest)); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to delete manifest %s: %v", manifest, err))
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func deleteDockerImageResource(client *http.Client, url string) error {
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get request: %v", err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete resource: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("failed to delete resource: status code = %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tr *GKETestRunner) createCluster(ctx context.Context, client *http.Client, projectID, zone, ClusterName string) error {
|
||||
request := &container.CreateClusterRequest{Cluster: &container.Cluster{
|
||||
Name: ClusterName,
|
||||
InitialNodeCount: 3,
|
||||
NodeConfig: &container.NodeConfig{
|
||||
OauthScopes: []string{
|
||||
storageReadScope,
|
||||
},
|
||||
},
|
||||
}}
|
||||
op, err := tr.ContainerService.Projects.Zones.Clusters.Create(projectID, zone, request).Context(ctx).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create cluster %s: %v", ClusterName, err)
|
||||
}
|
||||
opID := op.Name
|
||||
|
||||
// Wait for creating cluster.
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timed out waiting creating cluster")
|
||||
|
||||
case <-time.After(10 * time.Second):
|
||||
op, err := tr.ContainerService.Projects.Zones.Operations.Get(projectID, zone, opID).Context(ctx).Do()
|
||||
if err != nil {
|
||||
log.Printf("Transient error getting operation (will retry): %v", err)
|
||||
break
|
||||
}
|
||||
if op.Status == "DONE" {
|
||||
log.Printf("Created cluster %s.", ClusterName)
|
||||
return nil
|
||||
}
|
||||
if op.Status == "ABORTING" {
|
||||
return fmt.Errorf("create cluster operation is aborted")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tr *GKETestRunner) deployContainer(ctx context.Context, kubernetesClient *kubernetes.Client, podName, ImageName string) error {
|
||||
pod := &k8sapi.Pod{
|
||||
ObjectMeta: k8sapi.ObjectMeta{
|
||||
Name: podName,
|
||||
},
|
||||
Spec: k8sapi.PodSpec{
|
||||
Containers: []k8sapi.Container{
|
||||
{
|
||||
Name: "profiler-test",
|
||||
Image: fmt.Sprintf("gcr.io/%s:latest", ImageName),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if _, err := kubernetesClient.RunLongLivedPod(ctx, pod); err != nil {
|
||||
return fmt.Errorf("failed to run pod %s: %v", podName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PollPodLog polls the log of the kubernetes client and returns when the
|
||||
// finishString appears in the log, or when the context times out.
|
||||
func (tr *GKETestRunner) PollPodLog(ctx context.Context, kubernetesClient *kubernetes.Client, podName, finishString string) error {
|
||||
var output string
|
||||
defer func() {
|
||||
log.Printf("Log for pod %s:\n%s", podName, output)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("timed out waiting profiling finishing on container")
|
||||
|
||||
case <-time.After(20 * time.Second):
|
||||
var err error
|
||||
output, err = kubernetesClient.PodLog(ctx, podName)
|
||||
if err != nil {
|
||||
// Transient failure.
|
||||
log.Printf("Transient error getting log (will retry): %v", err)
|
||||
continue
|
||||
}
|
||||
if strings.Contains(output, finishString) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteClusterAndImage deletes cluster and images used to create cluster.
|
||||
func (tr *GKETestRunner) DeleteClusterAndImage(ctx context.Context, cfg *ClusterConfig) []error {
|
||||
var errs []error
|
||||
if err := tr.StorageClient.Bucket(cfg.Bucket).Object(cfg.ImageSourceName).Delete(ctx); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to delete storage client: %v", err))
|
||||
}
|
||||
for _, err := range tr.deleteDockerImage(ctx, cfg.ImageName) {
|
||||
errs = append(errs, fmt.Errorf("failed to delete docker image: %v", err))
|
||||
}
|
||||
if _, err := tr.ContainerService.Projects.Zones.Clusters.Delete(cfg.ProjectID, cfg.Zone, cfg.ClusterName).Context(ctx).Do(); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to delete cluster %s: %v", cfg.ClusterName, err))
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// StartAndDeployCluster creates image needed for cluster, then starts and
|
||||
// deploys to cluster.
|
||||
func (tr *GKETestRunner) StartAndDeployCluster(ctx context.Context, cfg *ClusterConfig) error {
|
||||
if err := tr.uploadImageSource(ctx, cfg.Bucket, cfg.ImageSourceName, cfg.Dockerfile); err != nil {
|
||||
return fmt.Errorf("failed to upload image source: %v", err)
|
||||
}
|
||||
|
||||
createImageCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := tr.createAndPublishDockerImage(createImageCtx, cfg.ProjectID, cfg.Bucket, cfg.ImageSourceName, fmt.Sprintf("gcr.io/%s", cfg.ImageName)); err != nil {
|
||||
return fmt.Errorf("failed to create and publish docker image %s: %v", cfg.ImageName, err)
|
||||
}
|
||||
|
||||
kubernetesClient, err := gke.NewClient(ctx, cfg.ClusterName, gke.OptZone(cfg.Zone), gke.OptProject(cfg.ProjectID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create new GKE client: %v", err)
|
||||
}
|
||||
|
||||
deployContainerCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
if err := tr.deployContainer(deployContainerCtx, kubernetesClient, cfg.PodName, cfg.ImageName); err != nil {
|
||||
return fmt.Errorf("failed to deploy image %q to pod %q: %v", cfg.PodName, cfg.ImageName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadImageSource uploads source code for building docker image to GCS.
|
||||
func (tr *GKETestRunner) uploadImageSource(ctx context.Context, bucket, objectName, dockerfile string) error {
|
||||
zipBuf := new(bytes.Buffer)
|
||||
z := zip.NewWriter(zipBuf)
|
||||
f, err := z.Create("Dockerfile")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := f.Write([]byte(dockerfile)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := z.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
wc := tr.StorageClient.Bucket(bucket).Object(objectName).NewWriter(ctx)
|
||||
wc.ContentType = "application/zip"
|
||||
wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
|
||||
if _, err := wc.Write(zipBuf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
return wc.Close()
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package profiler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/google/pprof/profile"
|
||||
)
|
||||
|
||||
var shouldAssumeSymbolized = isSymbolizedGoVersion(runtime.Version())
|
||||
|
||||
type function interface {
|
||||
Name() string
|
||||
FileLine(pc uintptr) (string, int)
|
||||
}
|
||||
|
||||
// funcForPC is a wrapper for runtime.FuncForPC. Defined as var for testing.
|
||||
var funcForPC = func(pc uintptr) function {
|
||||
if f := runtime.FuncForPC(pc); f != nil {
|
||||
return f
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseAndSymbolize parses a profile from a buffer, symbolizes it
|
||||
// if it's not yet symbolized, and writes the profile back as a
|
||||
// gzip-compressed marshaled protobuf.
|
||||
func parseAndSymbolize(data *bytes.Buffer) error {
|
||||
p, err := profile.ParseData(data.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do nothing if the profile is already symbolized.
|
||||
if symbolized(p) {
|
||||
return nil
|
||||
}
|
||||
// Clear the profile functions to avoid creating duplicates.
|
||||
p.Function = nil
|
||||
symbolize(p)
|
||||
data.Reset()
|
||||
return p.Write(data)
|
||||
}
|
||||
|
||||
// isSymbolizedGoVersion returns true if Go version equals to or is
|
||||
// higher than Go 1.9. Starting Go 1.9 the profiles are symbolized
|
||||
// by runtime/pprof.
|
||||
func isSymbolizedGoVersion(goVersion string) bool {
|
||||
r, err := regexp.Compile(`go(1\.9|1\.[1-9][0-9]|[2-9]).*`)
|
||||
if err == nil && r.MatchString(goVersion) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// symbolized checks if all locations have symbolized function
|
||||
// information.
|
||||
func symbolized(p *profile.Profile) bool {
|
||||
for _, l := range p.Location {
|
||||
if len(l.Line) == 0 || l.Line[0].Function == nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func symbolize(p *profile.Profile) {
|
||||
fns := profileFunctionMap{}
|
||||
for _, l := range p.Location {
|
||||
pc := uintptr(l.Address)
|
||||
f := funcForPC(pc)
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
file, lineno := f.FileLine(pc)
|
||||
l.Line = []profile.Line{
|
||||
{
|
||||
Function: fns.findOrAddFunction(f.Name(), file, p),
|
||||
Line: int64(lineno),
|
||||
},
|
||||
}
|
||||
}
|
||||
// Trim runtime functions. Always hide runtime.goexit. Other runtime
|
||||
// functions are only hidden for heap profile when they appear at the beginning.
|
||||
isHeapProfile := p.PeriodType != nil && p.PeriodType.Type == "space"
|
||||
for _, s := range p.Sample {
|
||||
show := !isHeapProfile
|
||||
var i int
|
||||
for _, l := range s.Location {
|
||||
if len(l.Line) > 0 && l.Line[0].Function != nil {
|
||||
name := l.Line[0].Function.Name
|
||||
if name == "runtime.goexit" || !show && strings.HasPrefix(name, "runtime.") {
|
||||
continue
|
||||
}
|
||||
}
|
||||
show = true
|
||||
s.Location[i] = l
|
||||
i++
|
||||
}
|
||||
// If all locations of a sample are trimmed, keep the root location.
|
||||
if i == 0 && len(s.Location) > 0 {
|
||||
s.Location[0] = s.Location[len(s.Location)-1]
|
||||
i = 1
|
||||
}
|
||||
s.Location = s.Location[:i]
|
||||
}
|
||||
}
|
||||
|
||||
type profileFunctionMap map[profile.Function]*profile.Function
|
||||
|
||||
func (fns profileFunctionMap) findOrAddFunction(name, filename string, p *profile.Profile) *profile.Function {
|
||||
f := profile.Function{
|
||||
Name: name,
|
||||
SystemName: name,
|
||||
Filename: filename,
|
||||
}
|
||||
if fp := fns[f]; fp != nil {
|
||||
return fp
|
||||
}
|
||||
fp := new(profile.Function)
|
||||
fns[f] = fp
|
||||
|
||||
*fp = f
|
||||
fp.ID = uint64(len(p.Function) + 1)
|
||||
p.Function = append(p.Function, fp)
|
||||
return fp
|
||||
}
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
// Copyright 2017 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package profiler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"cloud.google.com/go/internal/testutil"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/google/pprof/profile"
|
||||
)
|
||||
|
||||
type fakeFunc struct {
|
||||
name string
|
||||
file string
|
||||
lineno int
|
||||
}
|
||||
|
||||
func (f *fakeFunc) Name() string {
|
||||
return f.name
|
||||
}
|
||||
func (f *fakeFunc) FileLine(_ uintptr) (string, int) {
|
||||
return f.file, f.lineno
|
||||
}
|
||||
|
||||
var cmpOpt = cmpopts.IgnoreUnexported(profile.Profile{}, profile.Function{},
|
||||
profile.Line{}, profile.Location{}, profile.Sample{}, profile.ValueType{})
|
||||
|
||||
// TestRuntimeFunctionTrimming tests if symbolize trims runtime functions as intended.
|
||||
func TestRuntimeFunctionTrimming(t *testing.T) {
|
||||
fakeFuncMap := map[uintptr]*fakeFunc{
|
||||
0x10: &fakeFunc{"runtime.goexit", "runtime.go", 10},
|
||||
0x20: &fakeFunc{"runtime.other", "runtime.go", 20},
|
||||
0x30: &fakeFunc{"foo", "foo.go", 30},
|
||||
0x40: &fakeFunc{"bar", "bar.go", 40},
|
||||
}
|
||||
backupFuncForPC := funcForPC
|
||||
funcForPC = func(pc uintptr) function {
|
||||
return fakeFuncMap[pc]
|
||||
}
|
||||
defer func() {
|
||||
funcForPC = backupFuncForPC
|
||||
}()
|
||||
testLoc := []*profile.Location{
|
||||
{ID: 1, Address: 0x10},
|
||||
{ID: 2, Address: 0x20},
|
||||
{ID: 3, Address: 0x30},
|
||||
{ID: 4, Address: 0x40},
|
||||
}
|
||||
testProfile := &profile.Profile{
|
||||
Sample: []*profile.Sample{
|
||||
{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[2]}},
|
||||
{Location: []*profile.Location{testLoc[1], testLoc[3], testLoc[2]}},
|
||||
{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[1]}},
|
||||
{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[0]}},
|
||||
{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[0]}},
|
||||
{Location: []*profile.Location{testLoc[1], testLoc[0]}},
|
||||
},
|
||||
Location: testLoc,
|
||||
}
|
||||
testProfiles := make([]*profile.Profile, 2)
|
||||
testProfiles[0] = testProfile.Copy()
|
||||
testProfiles[1] = testProfile.Copy()
|
||||
// Test case for CPU profile.
|
||||
testProfiles[0].PeriodType = &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
|
||||
// Test case for heap profile.
|
||||
testProfiles[1].PeriodType = &profile.ValueType{Type: "space", Unit: "bytes"}
|
||||
wantFunc := []*profile.Function{
|
||||
{ID: 1, Name: "runtime.goexit", SystemName: "runtime.goexit", Filename: "runtime.go"},
|
||||
{ID: 2, Name: "runtime.other", SystemName: "runtime.other", Filename: "runtime.go"},
|
||||
{ID: 3, Name: "foo", SystemName: "foo", Filename: "foo.go"},
|
||||
{ID: 4, Name: "bar", SystemName: "bar", Filename: "bar.go"},
|
||||
}
|
||||
wantLoc := []*profile.Location{
|
||||
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
|
||||
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
|
||||
{ID: 3, Address: 0x30, Line: []profile.Line{{Function: wantFunc[2], Line: 30}}},
|
||||
{ID: 4, Address: 0x40, Line: []profile.Line{{Function: wantFunc[3], Line: 40}}},
|
||||
}
|
||||
wantProfiles := []*profile.Profile{
|
||||
{
|
||||
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
Sample: []*profile.Sample{
|
||||
{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[1], wantLoc[3]}},
|
||||
{Location: []*profile.Location{wantLoc[1]}},
|
||||
},
|
||||
Location: wantLoc,
|
||||
Function: wantFunc,
|
||||
},
|
||||
{
|
||||
PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},
|
||||
Sample: []*profile.Sample{
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
|
||||
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
|
||||
{Location: []*profile.Location{wantLoc[3]}},
|
||||
{Location: []*profile.Location{wantLoc[0]}},
|
||||
},
|
||||
Location: wantLoc,
|
||||
Function: wantFunc,
|
||||
},
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
symbolize(testProfiles[i])
|
||||
if !testutil.Equal(testProfiles[i], wantProfiles[i], cmpOpt) {
|
||||
t.Errorf("incorrect trimming (testcase = %d): got {%v}, want {%v}", i, testProfiles[i], wantProfiles[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseAndSymbolize tests if parseAndSymbolize parses and symbolizes
|
||||
// profiles as intended.
|
||||
func TestParseAndSymbolize(t *testing.T) {
|
||||
fakeFuncMap := map[uintptr]*fakeFunc{
|
||||
0x10: &fakeFunc{"foo", "foo.go", 10},
|
||||
0x20: &fakeFunc{"bar", "bar.go", 20},
|
||||
}
|
||||
backupFuncForPC := funcForPC
|
||||
funcForPC = func(pc uintptr) function {
|
||||
return fakeFuncMap[pc]
|
||||
}
|
||||
defer func() {
|
||||
funcForPC = backupFuncForPC
|
||||
}()
|
||||
|
||||
testLoc := []*profile.Location{
|
||||
{ID: 1, Address: 0x10},
|
||||
{ID: 2, Address: 0x20},
|
||||
}
|
||||
testProfile := &profile.Profile{
|
||||
SampleType: []*profile.ValueType{
|
||||
&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
},
|
||||
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
Sample: []*profile.Sample{
|
||||
{Location: []*profile.Location{testLoc[0], testLoc[1]}, Value: []int64{1}},
|
||||
{Location: []*profile.Location{testLoc[1]}, Value: []int64{1}},
|
||||
},
|
||||
Location: testLoc,
|
||||
}
|
||||
testProfiles := make([]*profile.Profile, 2)
|
||||
testProfiles[0] = testProfile.Copy()
|
||||
testProfiles[1] = testProfile.Copy()
|
||||
|
||||
wantFunc := []*profile.Function{
|
||||
{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
|
||||
{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
|
||||
}
|
||||
wantLoc := []*profile.Location{
|
||||
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
|
||||
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
|
||||
}
|
||||
wantProfile := &profile.Profile{
|
||||
SampleType: []*profile.ValueType{
|
||||
&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
},
|
||||
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
|
||||
Sample: []*profile.Sample{
|
||||
{Location: []*profile.Location{wantLoc[0], wantLoc[1]}, Value: []int64{1}},
|
||||
{Location: []*profile.Location{wantLoc[1]}, Value: []int64{1}},
|
||||
},
|
||||
Location: wantLoc,
|
||||
Function: wantFunc,
|
||||
}
|
||||
|
||||
// Profile already symbolized.
|
||||
testProfiles[1].Location = []*profile.Location{
|
||||
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
|
||||
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
|
||||
}
|
||||
testProfiles[1].Function = []*profile.Function{
|
||||
{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
|
||||
{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
var prof bytes.Buffer
|
||||
testProfiles[i].Write(&prof)
|
||||
|
||||
parseAndSymbolize(&prof)
|
||||
gotProfile, err := profile.ParseData(prof.Bytes())
|
||||
if err != nil {
|
||||
t.Errorf("parsing symbolized profile (testcase = %d) got err: %v, want no error", i, err)
|
||||
}
|
||||
if !testutil.Equal(gotProfile, wantProfile, cmpOpt) {
|
||||
t.Errorf("incorrect symbolization (testcase = %d): got {%v}, want {%v}", i, gotProfile, wantProfile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSymbolizedGoVersion(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{"go1.9beta2", true},
|
||||
{"go1.9", true},
|
||||
{"go1.9.1", true},
|
||||
{"go1.10", true},
|
||||
{"go1.10.1", true},
|
||||
{"go2.0", true},
|
||||
{"go3.1", true},
|
||||
{"go1.8", false},
|
||||
{"go1.8.1", false},
|
||||
{"go1.7", false},
|
||||
{"devel ", false},
|
||||
} {
|
||||
if got := isSymbolizedGoVersion(tc.input); got != tc.want {
|
||||
t.Errorf("isSymbolizedGoVersion(%v) got %v, want %v", tc.input, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user