initial commit

Signed-off-by: Derek McQuay <derekmcquay@gmail.com>
This commit is contained in:
dm
2018-04-10 19:17:26 -07:00
commit 59af8fc84f
2025 changed files with 456739 additions and 0 deletions
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"cloud.google.com/go/profiler"
"compress/gzip"
"flag"
"log"
"math/rand"
"sync"
"time"
)
var (
service = flag.String("service", "", "service name")
mutexProfiling = flag.Bool("mutex_profiling", false, "enable mutex profiling")
duration = flag.Int("duration", 600, "duration of the benchmark in seconds")
apiAddr = flag.String("api_address", "", "API address of the profiler (e.g. 'cloudprofiler.googleapis.com:443')")
)
// busywork continuously generates 1MiB of random data and compresses it
// throwing away the result.
func busywork(mu *sync.Mutex) {
ticker := time.NewTicker(time.Duration(*duration) * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
return
default:
mu.Lock()
busyworkOnce()
mu.Unlock()
}
}
}
func busyworkOnce() {
data := make([]byte, 1024*1024)
rand.Read(data)
var b bytes.Buffer
gz := gzip.NewWriter(&b)
if _, err := gz.Write(data); err != nil {
log.Printf("Failed to write to gzip stream: %v", err)
return
}
if err := gz.Flush(); err != nil {
log.Printf("Failed to flush to gzip stream: %v", err)
return
}
if err := gz.Close(); err != nil {
log.Printf("Failed to close gzip stream: %v", err)
}
// Throw away the result.
}
func main() {
flag.Parse()
if *service == "" {
log.Print("Service name must be configured using --service flag.")
} else if err := profiler.Start(
profiler.Config{
Service: *service,
MutexProfiling: *mutexProfiling,
DebugLogging: true,
APIAddr: *apiAddr,
}); err != nil {
log.Printf("Failed to start the profiler: %v", err)
} else {
mu := new(sync.Mutex)
var wg sync.WaitGroup
wg.Add(5)
for i := 0; i < 5; i++ {
go func() {
defer wg.Done()
busywork(mu)
}()
}
wg.Wait()
}
log.Printf("busybench finished profiling.")
// Do not exit, since the pod in the GKE test is set to always restart.
select {}
}
+36
View File
@@ -0,0 +1,36 @@
#!/bin/bash
# Fail on any error.
set -eo pipefail
# Display commands being run.
set -x
cd git/gocloud
COMMIT=$(git rev-parse HEAD)
# Set $GOPATH
export GOPATH="$HOME/go"
GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
mkdir -p $GOCLOUD_HOME
# Move code into $GOPATH and get dependencies
cp -R ./* $GOCLOUD_HOME
cd $GOCLOUD_HOME/internal/kokoro
# Don't print out encryption keys, etc
set +x
key=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_key")
iv=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_iv")
pass=$(cat "$KOKORO_ARTIFACTS_DIR/keystore/72523_encrypted_ba2d6f7723ed_pass")
openssl aes-256-cbc -K $key -iv $iv -pass pass:$pass -in kokoro-key.json.enc -out key.json -d
set -x
export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/key.json"
export GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762"
export GCLOUD_TESTS_GOLANG_ZONE="us-west1-a"
export GCLOUD_TESTS_GOLANG_BUCKET="dulcet-port-762-go-cloud-profiler-test"
cd $GOCLOUD_HOME/profiler
go get -t -tags=integration .
go test -timeout=60m -parallel=5 -tags=integration -run TestAgentIntegration -commit="$COMMIT"
+266
View File
@@ -0,0 +1,266 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build integration,go1.7
package profiler
import (
"bytes"
"flag"
"fmt"
"os"
"testing"
"text/template"
"time"
"cloud.google.com/go/profiler/proftest"
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
compute "google.golang.org/api/compute/v1"
)
var (
commit = flag.String("commit", "", "git commit to test")
runID = time.Now().Unix()
)
const (
cloudScope = "https://www.googleapis.com/auth/cloud-platform"
benchFinishString = "busybench finished profiling"
)
const startupTemplate = `
#! /bin/bash
(
# Shut down the VM in 5 minutes after this script exits
# to stop accounting the VM for billing and cores quota.
trap "sleep 300 && poweroff" EXIT
# Fail on any error.
set -eo pipefail
# Display commands being run.
set -x
# Install git
apt-get update >/dev/null
apt-get -y -q install git >/dev/null
# Install desired Go version
mkdir -p /tmp/bin
curl -sL -o /tmp/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
chmod +x /tmp/bin/gimme
export PATH=$PATH:/tmp/bin
eval "$(gimme {{.GoVersion}})"
# Set $GOPATH
export GOPATH="$HOME/go"
export GOCLOUD_HOME=$GOPATH/src/cloud.google.com/go
mkdir -p $GOCLOUD_HOME
# Install agent
git clone https://code.googlesource.com/gocloud $GOCLOUD_HOME >/dev/null
cd $GOCLOUD_HOME/profiler/busybench
git reset --hard {{.Commit}}
go get -v >/dev/null
# Run benchmark with agent
go run busybench.go --service="{{.Service}}" --mutex_profiling="{{.MutexProfiling}}"
# Write output to serial port 2 with timestamp.
) 2>&1 | while read line; do echo "$(date): ${line}"; done >/dev/ttyS1
`
const dockerfileFmt = `FROM golang
RUN git clone https://code.googlesource.com/gocloud /go/src/cloud.google.com/go \
&& cd /go/src/cloud.google.com/go/profiler/busybench && git reset --hard %s \
&& go get -v && go install -v
CMD ["busybench", "--service", "%s"]
`
type goGCETestCase struct {
proftest.InstanceConfig
name string
goVersion string
mutexProfiling bool
wantProfileTypes []string
}
func (tc *goGCETestCase) initializeStartupScript(template *template.Template) error {
var buf bytes.Buffer
err := template.Execute(&buf,
struct {
Service string
GoVersion string
Commit string
MutexProfiling bool
}{
Service: tc.name,
GoVersion: tc.goVersion,
Commit: *commit,
MutexProfiling: tc.mutexProfiling,
})
if err != nil {
return fmt.Errorf("failed to render startup script for %s: %v", tc.name, err)
}
tc.StartupScript = buf.String()
return nil
}
func TestAgentIntegration(t *testing.T) {
projectID := os.Getenv("GCLOUD_TESTS_GOLANG_PROJECT_ID")
if projectID == "" {
t.Fatalf("Getenv(GCLOUD_TESTS_GOLANG_PROJECT_ID) got empty string")
}
zone := os.Getenv("GCLOUD_TESTS_GOLANG_ZONE")
if zone == "" {
t.Fatalf("Getenv(GCLOUD_TESTS_GOLANG_ZONE) got empty string")
}
if *commit == "" {
t.Fatal("commit flag is not set")
}
ctx := context.Background()
client, err := google.DefaultClient(ctx, cloudScope)
if err != nil {
t.Fatalf("failed to get default client: %v", err)
}
computeService, err := compute.New(client)
if err != nil {
t.Fatalf("failed to initialize compute service: %v", err)
}
template, err := template.New("startupScript").Parse(startupTemplate)
if err != nil {
t.Fatalf("failed to parse startup script template: %v", err)
}
tr := proftest.TestRunner{
Client: client,
}
gceTr := proftest.GCETestRunner{
TestRunner: tr,
ComputeService: computeService,
}
testcases := []goGCETestCase{
{
InstanceConfig: proftest.InstanceConfig{
ProjectID: projectID,
Zone: zone,
Name: fmt.Sprintf("profiler-test-go110-%d", runID),
MachineType: "n1-standard-1",
},
name: fmt.Sprintf("profiler-test-go110-%d-gce", runID),
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
goVersion: "1.10",
mutexProfiling: true,
},
{
InstanceConfig: proftest.InstanceConfig{
ProjectID: projectID,
Zone: zone,
Name: fmt.Sprintf("profiler-test-go19-%d", runID),
MachineType: "n1-standard-1",
},
name: fmt.Sprintf("profiler-test-go19-%d-gce", runID),
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
goVersion: "1.9",
mutexProfiling: true,
},
{
InstanceConfig: proftest.InstanceConfig{
ProjectID: projectID,
Zone: zone,
Name: fmt.Sprintf("profiler-test-go18-%d", runID),
MachineType: "n1-standard-1",
},
name: fmt.Sprintf("profiler-test-go18-%d-gce", runID),
wantProfileTypes: []string{"CPU", "HEAP", "THREADS", "CONTENTION"},
goVersion: "1.8",
mutexProfiling: true,
},
{
InstanceConfig: proftest.InstanceConfig{
ProjectID: projectID,
Zone: zone,
Name: fmt.Sprintf("profiler-test-go17-%d", runID),
MachineType: "n1-standard-1",
},
name: fmt.Sprintf("profiler-test-go17-%d-gce", runID),
wantProfileTypes: []string{"CPU", "HEAP", "THREADS"},
goVersion: "1.7",
},
{
InstanceConfig: proftest.InstanceConfig{
ProjectID: projectID,
Zone: zone,
Name: fmt.Sprintf("profiler-test-go16-%d", runID),
MachineType: "n1-standard-1",
},
name: fmt.Sprintf("profiler-test-go16-%d-gce", runID),
wantProfileTypes: []string{"CPU", "HEAP", "THREADS"},
goVersion: "1.6",
},
}
for _, tc := range testcases {
tc := tc // capture range variable
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if err := tc.initializeStartupScript(template); err != nil {
t.Fatalf("failed to initialize startup script")
}
if err := gceTr.StartInstance(ctx, &tc.InstanceConfig); err != nil {
t.Fatal(err)
}
defer func() {
if gceTr.DeleteInstance(ctx, &tc.InstanceConfig); err != nil {
t.Fatal(err)
}
}()
timeoutCtx, cancel := context.WithTimeout(ctx, time.Minute*25)
defer cancel()
if err := gceTr.PollForSerialOutput(timeoutCtx, &tc.InstanceConfig, benchFinishString); err != nil {
t.Fatal(err)
}
timeNow := time.Now()
endTime := timeNow.Format(time.RFC3339)
startTime := timeNow.Add(-1 * time.Hour).Format(time.RFC3339)
for _, pType := range tc.wantProfileTypes {
pr, err := tr.QueryProfiles(tc.ProjectID, tc.name, startTime, endTime, pType)
if err != nil {
t.Errorf("QueryProfiles(%s, %s, %s, %s, %s) got error: %v", tc.ProjectID, tc.name, startTime, endTime, pType, err)
continue
}
if err := pr.HasFunction("busywork"); err != nil {
t.Error(err)
}
}
})
}
}
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Automatically generated by MockGen. DO NOT EDIT!
// Source: google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2 (interfaces: ProfilerServiceClient)
package mocks
import (
gomock "github.com/golang/mock/gomock"
context "golang.org/x/net/context"
v2 "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
grpc "google.golang.org/grpc"
)
// Mock of ProfilerServiceClient interface
type MockProfilerServiceClient struct {
ctrl *gomock.Controller
recorder *_MockProfilerServiceClientRecorder
}
// Recorder for MockProfilerServiceClient (not exported)
type _MockProfilerServiceClientRecorder struct {
mock *MockProfilerServiceClient
}
func NewMockProfilerServiceClient(ctrl *gomock.Controller) *MockProfilerServiceClient {
mock := &MockProfilerServiceClient{ctrl: ctrl}
mock.recorder = &_MockProfilerServiceClientRecorder{mock}
return mock
}
func (_m *MockProfilerServiceClient) EXPECT() *_MockProfilerServiceClientRecorder {
return _m.recorder
}
func (_m *MockProfilerServiceClient) CreateProfile(_param0 context.Context, _param1 *v2.CreateProfileRequest, _param2 ...grpc.CallOption) (*v2.Profile, error) {
_s := []interface{}{_param0, _param1}
for _, _x := range _param2 {
_s = append(_s, _x)
}
ret := _m.ctrl.Call(_m, "CreateProfile", _s...)
ret0, _ := ret[0].(*v2.Profile)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockProfilerServiceClientRecorder) CreateProfile(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
_s := append([]interface{}{arg0, arg1}, arg2...)
return _mr.mock.ctrl.RecordCall(_mr.mock, "CreateProfile", _s...)
}
func (_m *MockProfilerServiceClient) UpdateProfile(_param0 context.Context, _param1 *v2.UpdateProfileRequest, _param2 ...grpc.CallOption) (*v2.Profile, error) {
_s := []interface{}{_param0, _param1}
for _, _x := range _param2 {
_s = append(_s, _x)
}
ret := _m.ctrl.Call(_m, "UpdateProfile", _s...)
ret0, _ := ret[0].(*v2.Profile)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockProfilerServiceClientRecorder) UpdateProfile(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call {
_s := append([]interface{}{arg0, arg1}, arg2...)
return _mr.mock.ctrl.RecordCall(_mr.mock, "UpdateProfile", _s...)
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package profiler
import "runtime"
func enableMutexProfiling() bool {
// One percent of mutex contention events are profiled.
runtime.SetMutexProfileFraction(100)
return true
}
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.8
package profiler
func enableMutexProfiling() bool {
return false
}
+506
View File
@@ -0,0 +1,506 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package profiler is a client for the Stackdriver Profiler service.
//
// This package is still experimental and subject to change.
//
// Usage example:
//
// import "cloud.google.com/go/profiler"
// ...
// if err := profiler.Start(profiler.Config{Service: "my-service"}); err != nil {
// // TODO: Handle error.
// }
//
// Calling Start will start a goroutine to collect profiles and upload to
// the profiler server, at the rhythm specified by the server.
//
// The caller must provide the service string in the config, and may provide
// other information as well. See Config for details.
//
// Profiler has CPU, heap and goroutine profiling enabled by default. Mutex
// profiling can be enabled in the config. Note that goroutine and mutex
// profiles are shown as "threads" and "contention" profiles in the profiler
// UI.
package profiler
import (
"bytes"
"errors"
"fmt"
"log"
"os"
"runtime"
"runtime/pprof"
"sync"
"time"
gcemd "cloud.google.com/go/compute/metadata"
"cloud.google.com/go/internal/version"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/google/pprof/profile"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
"google.golang.org/api/option"
gtransport "google.golang.org/api/transport/grpc"
pb "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
edpb "google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
grpcmd "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
var (
config Config
startOnce sync.Once
mutexEnabled bool
// The functions below are stubbed to be overrideable for testing.
getProjectID = gcemd.ProjectID
getInstanceName = gcemd.InstanceName
getZone = gcemd.Zone
startCPUProfile = pprof.StartCPUProfile
stopCPUProfile = pprof.StopCPUProfile
writeHeapProfile = pprof.WriteHeapProfile
sleep = gax.Sleep
dialGRPC = gtransport.Dial
onGCE = gcemd.OnGCE
)
const (
apiAddress = "cloudprofiler.googleapis.com:443"
xGoogAPIMetadata = "x-goog-api-client"
zoneNameLabel = "zone"
versionLabel = "version"
instanceLabel = "instance"
scope = "https://www.googleapis.com/auth/monitoring.write"
initialBackoff = time.Second
// Ensure the agent will recover within 1 hour.
maxBackoff = time.Hour
backoffMultiplier = 1.3 // Backoff envelope increases by this factor on each retry.
retryInfoMetadata = "google.rpc.retryinfo-bin"
)
// Config is the profiler configuration.
type Config struct {
// Service (or deprecated Target) must be provided to start the profiler.
// It specifies the name of the service under which the profiled data
// will be recorded and exposed at the Profiler UI for the project.
// You can specify an arbitrary string, but see Deployment.target at
// https://github.com/googleapis/googleapis/blob/master/google/devtools/cloudprofiler/v2/profiler.proto
// for restrictions.
// NOTE: The string should be the same across different replicas of
// your service so that the globally constant profiling rate is
// maintained. Do not put things like PID or unique pod ID in the name.
Service string
// ServiceVersion is an optional field specifying the version of the
// service. It can be an arbitrary string. Profiler profiles
// once per minute for each version of each service in each zone.
// ServiceVersion defaults to an empty string.
ServiceVersion string
// DebugLogging enables detailed debug logging from profiler. It
// defaults to false.
DebugLogging bool
// MutexProfiling enables mutex profiling. It defaults to false.
// Note that mutex profiling is not supported by Go versions older
// than Go 1.8.
MutexProfiling bool
// When true, collecting the heap profiles is disabled.
NoHeapProfiling bool
// When true, collecting the goroutine profiles is disabled.
NoGoroutineProfiling bool
// ProjectID is the Cloud Console project ID to use instead of
// the one read from the VM metadata server.
//
// Set this if you are running the agent in your local environment
// or anywhere else outside of Google Cloud Platform.
ProjectID string
// APIAddr is the HTTP endpoint to use to connect to the profiler
// agent API. Defaults to the production environment, overridable
// for testing.
APIAddr string
// Target is deprecated, use Service instead.
Target string
instance string
zone string
}
// startError represents the error occured during the
// initializating and starting of the agent.
var startError error
// Start starts a goroutine to collect and upload profiles. The
// caller must provide the service string in the config. See
// Config for details. Start should only be called once. Any
// additional calls will be ignored.
func Start(cfg Config, options ...option.ClientOption) error {
startOnce.Do(func() {
startError = start(cfg, options...)
})
return startError
}
func start(cfg Config, options ...option.ClientOption) error {
if err := initializeConfig(cfg); err != nil {
debugLog("failed to initialize config: %v", err)
return err
}
if config.MutexProfiling {
if mutexEnabled = enableMutexProfiling(); !mutexEnabled {
return fmt.Errorf("mutex profiling is not supported by %s, requires Go 1.8 or later", runtime.Version())
}
}
ctx := context.Background()
opts := []option.ClientOption{
option.WithEndpoint(config.APIAddr),
option.WithScopes(scope),
}
opts = append(opts, options...)
conn, err := dialGRPC(ctx, opts...)
if err != nil {
debugLog("failed to dial GRPC: %v", err)
return err
}
a := initializeAgent(pb.NewProfilerServiceClient(conn))
go pollProfilerService(withXGoogHeader(ctx), a)
return nil
}
func debugLog(format string, e ...interface{}) {
if config.DebugLogging {
log.Printf(format, e...)
}
}
// agent polls the profiler server for instructions on behalf of a task,
// and collects and uploads profiles as requested.
type agent struct {
client pb.ProfilerServiceClient
deployment *pb.Deployment
profileLabels map[string]string
profileTypes []pb.ProfileType
}
// abortedBackoffDuration retrieves the retry duration from gRPC trailing
// metadata, which is set by the profiler server.
func abortedBackoffDuration(md grpcmd.MD) (time.Duration, error) {
elem := md[retryInfoMetadata]
if len(elem) <= 0 {
return 0, errors.New("no retry info")
}
var retryInfo edpb.RetryInfo
if err := proto.Unmarshal([]byte(elem[0]), &retryInfo); err != nil {
return 0, err
} else if time, err := ptypes.Duration(retryInfo.RetryDelay); err != nil {
return 0, err
} else {
if time < 0 {
return 0, errors.New("negative retry duration")
}
return time, nil
}
}
type retryer struct {
backoff gax.Backoff
md grpcmd.MD
}
func (r *retryer) Retry(err error) (time.Duration, bool) {
st, _ := status.FromError(err)
if st != nil && st.Code() == codes.Aborted {
dur, err := abortedBackoffDuration(r.md)
if err == nil {
return dur, true
}
debugLog("failed to get backoff duration: %v", err)
}
return r.backoff.Pause(), true
}
// createProfile talks to the profiler server to create profile. In
// case of error, the goroutine will sleep and retry. Sleep duration may
// be specified by the server. Otherwise it will be an exponentially
// increasing value, bounded by maxBackoff.
func (a *agent) createProfile(ctx context.Context) *pb.Profile {
req := pb.CreateProfileRequest{
Deployment: a.deployment,
ProfileType: a.profileTypes,
}
var p *pb.Profile
md := grpcmd.New(map[string]string{})
gax.Invoke(ctx, func(ctx context.Context, settings gax.CallSettings) error {
var err error
p, err = a.client.CreateProfile(ctx, &req, grpc.Trailer(&md))
if err != nil {
debugLog("failed to create a profile, will retry: %v", err)
}
return err
}, gax.WithRetry(func() gax.Retryer {
return &retryer{
backoff: gax.Backoff{
Initial: initialBackoff,
Max: maxBackoff,
Multiplier: backoffMultiplier,
},
md: md,
}
}))
debugLog("successfully created profile %v", p.GetProfileType())
return p
}
func (a *agent) profileAndUpload(ctx context.Context, p *pb.Profile) {
var prof bytes.Buffer
pt := p.GetProfileType()
switch pt {
case pb.ProfileType_CPU:
duration, err := ptypes.Duration(p.Duration)
if err != nil {
debugLog("failed to get profile duration: %v", err)
return
}
if err := startCPUProfile(&prof); err != nil {
debugLog("failed to start CPU profile: %v", err)
return
}
sleep(ctx, duration)
stopCPUProfile()
case pb.ProfileType_HEAP:
if err := writeHeapProfile(&prof); err != nil {
debugLog("failed to write heap profile: %v", err)
return
}
case pb.ProfileType_THREADS:
if err := pprof.Lookup("goroutine").WriteTo(&prof, 0); err != nil {
debugLog("failed to create goroutine profile: %v", err)
return
}
case pb.ProfileType_CONTENTION:
duration, err := ptypes.Duration(p.Duration)
if err != nil {
debugLog("failed to get profile duration: %v", err)
return
}
if err := deltaMutexProfile(ctx, duration, &prof); err != nil {
debugLog("failed to create mutex profile: %v", err)
return
}
default:
debugLog("unexpected profile type: %v", pt)
return
}
// Starting Go 1.9 the profiles are symbolized by runtime/pprof.
// TODO(jianqiaoli): Remove the symbolization code when we decide to
// stop supporting Go 1.8.
if !shouldAssumeSymbolized && pt != pb.ProfileType_CONTENTION {
if err := parseAndSymbolize(&prof); err != nil {
debugLog("failed to symbolize profile: %v", err)
}
}
p.ProfileBytes = prof.Bytes()
p.Labels = a.profileLabels
req := pb.UpdateProfileRequest{Profile: p}
// Upload profile, discard profile in case of error.
debugLog("start uploading profile")
if _, err := a.client.UpdateProfile(ctx, &req); err != nil {
debugLog("failed to upload profile: %v", err)
}
}
// deltaMutexProfile writes mutex profile changes over a time period specified
// with 'duration' to 'prof'.
func deltaMutexProfile(ctx context.Context, duration time.Duration, prof *bytes.Buffer) error {
if !mutexEnabled {
return errors.New("mutex profiling is not enabled")
}
p0, err := mutexProfile()
if err != nil {
return err
}
sleep(ctx, duration)
p, err := mutexProfile()
if err != nil {
return err
}
// TODO(jianqiaoli): Remove this check when github.com/google/pprof/issues/242
// is fixed.
if len(p0.Mapping) > 0 {
p0.Scale(-1)
p, err = profile.Merge([]*profile.Profile{p0, p})
if err != nil {
return err
}
}
// The mutex profile is not symbolized by runtime.pprof until
// golang.org/issue/21474 is fixed in go1.10.
symbolize(p)
return p.Write(prof)
}
func mutexProfile() (*profile.Profile, error) {
p := pprof.Lookup("mutex")
if p == nil {
return nil, errors.New("mutex profiling is not supported")
}
var buf bytes.Buffer
if err := p.WriteTo(&buf, 0); err != nil {
return nil, err
}
return profile.Parse(&buf)
}
// withXGoogHeader sets the name and version of the application in
// the `x-goog-api-client` header passed on each request. Intended for
// use by Google-written clients.
func withXGoogHeader(ctx context.Context, keyval ...string) context.Context {
kv := append([]string{"gl-go", version.Go(), "gccl", version.Repo}, keyval...)
kv = append(kv, "gax", gax.Version, "grpc", grpc.Version)
md, _ := grpcmd.FromOutgoingContext(ctx)
md = md.Copy()
md[xGoogAPIMetadata] = []string{gax.XGoogHeader(kv...)}
return grpcmd.NewOutgoingContext(ctx, md)
}
func initializeAgent(c pb.ProfilerServiceClient) *agent {
labels := map[string]string{}
if config.zone != "" {
labels[zoneNameLabel] = config.zone
}
if config.ServiceVersion != "" {
labels[versionLabel] = config.ServiceVersion
}
d := &pb.Deployment{
ProjectId: config.ProjectID,
Target: config.Target,
Labels: labels,
}
profileLabels := map[string]string{}
if config.instance != "" {
profileLabels[instanceLabel] = config.instance
}
profileTypes := []pb.ProfileType{pb.ProfileType_CPU}
if !config.NoHeapProfiling {
profileTypes = append(profileTypes, pb.ProfileType_HEAP)
}
if !config.NoGoroutineProfiling {
profileTypes = append(profileTypes, pb.ProfileType_THREADS)
}
if mutexEnabled {
profileTypes = append(profileTypes, pb.ProfileType_CONTENTION)
}
return &agent{
client: c,
deployment: d,
profileLabels: profileLabels,
profileTypes: profileTypes,
}
}
func initializeConfig(cfg Config) error {
config = cfg
switch {
case config.Service != "":
config.Target = config.Service
case config.Target == "":
config.Target = os.Getenv("GAE_SERVICE")
}
if config.Target == "" {
return errors.New("service name must be specified in the configuration")
}
if config.ServiceVersion == "" {
config.ServiceVersion = os.Getenv("GAE_VERSION")
}
if projectID := os.Getenv("GOOGLE_CLOUD_PROJECT"); config.ProjectID == "" && projectID != "" {
// Cloud Shell and App Engine set this environment variable to the project
// ID, so use it if present. In case of App Engine the project ID is also
// available from the GCE metadata server, but by using the environment
// variable saves one request to the metadata server. The environment
// project ID is only used if no project ID is provided in the
// configuration.
config.ProjectID = projectID
}
if onGCE() {
var err error
if config.ProjectID == "" {
if config.ProjectID, err = getProjectID(); err != nil {
return fmt.Errorf("failed to get the project ID from Compute Engine: %v", err)
}
}
if config.zone, err = getZone(); err != nil {
return fmt.Errorf("failed to get zone from Compute Engine: %v", err)
}
if config.instance, err = getInstanceName(); err != nil {
return fmt.Errorf("failed to get instance from Compute Engine: %v", err)
}
} else {
if config.ProjectID == "" {
return fmt.Errorf("project ID must be specified in the configuration if running outside of GCP")
}
}
if config.APIAddr == "" {
config.APIAddr = apiAddress
}
return nil
}
// pollProfilerService starts an endless loop to poll the profiler
// server for instructions, and collects and uploads profiles as
// requested.
func pollProfilerService(ctx context.Context, a *agent) {
debugLog("profiler has started")
for {
p := a.createProfile(ctx)
a.profileAndUpload(ctx, p)
}
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package profiler_test
import (
"cloud.google.com/go/profiler"
)
func ExampleStart() {
if err := profiler.Start(profiler.Config{Service: "my-service", ServiceVersion: "v1"}); err != nil {
//TODO: Handle error.
}
}
+876
View File
@@ -0,0 +1,876 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package profiler
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"io"
"log"
"math/rand"
"os"
"runtime"
"strings"
"sync"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/profiler/mocks"
"github.com/golang/mock/gomock"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/google/pprof/profile"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
gtransport "google.golang.org/api/transport/grpc"
pb "google.golang.org/genproto/googleapis/devtools/cloudprofiler/v2"
edpb "google.golang.org/genproto/googleapis/rpc/errdetails"
"google.golang.org/grpc/codes"
grpcmd "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
const (
testProjectID = "test-project-ID"
testInstance = "test-instance"
testZone = "test-zone"
testTarget = "test-target"
testService = "test-service"
testSvcVersion = "test-service-version"
testProfileDuration = time.Second * 10
testServerTimeout = time.Second * 15
)
func createTestDeployment() *pb.Deployment {
labels := map[string]string{
zoneNameLabel: testZone,
versionLabel: testSvcVersion,
}
return &pb.Deployment{
ProjectId: testProjectID,
Target: testService,
Labels: labels,
}
}
func createTestAgent(psc pb.ProfilerServiceClient) *agent {
return &agent{
client: psc,
deployment: createTestDeployment(),
profileLabels: map[string]string{instanceLabel: testInstance},
profileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
}
}
func createTrailers(dur time.Duration) map[string]string {
b, _ := proto.Marshal(&edpb.RetryInfo{
RetryDelay: ptypes.DurationProto(dur),
})
return map[string]string{
retryInfoMetadata: string(b),
}
}
func TestCreateProfile(t *testing.T) {
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mpc := mocks.NewMockProfilerServiceClient(ctrl)
a := createTestAgent(mpc)
p := &pb.Profile{Name: "test_profile"}
wantRequest := pb.CreateProfileRequest{
Deployment: a.deployment,
ProfileType: a.profileTypes,
}
mpc.EXPECT().CreateProfile(ctx, gomock.Eq(&wantRequest), gomock.Any()).Times(1).Return(p, nil)
gotP := a.createProfile(ctx)
if !testutil.Equal(gotP, p) {
t.Errorf("CreateProfile() got wrong profile, got %v, want %v", gotP, p)
}
}
func TestProfileAndUpload(t *testing.T) {
oldStartCPUProfile, oldStopCPUProfile, oldWriteHeapProfile, oldSleep := startCPUProfile, stopCPUProfile, writeHeapProfile, sleep
defer func() {
startCPUProfile, stopCPUProfile, writeHeapProfile, sleep = oldStartCPUProfile, oldStopCPUProfile, oldWriteHeapProfile, oldSleep
}()
ctx := context.Background()
ctrl := gomock.NewController(t)
defer ctrl.Finish()
errFunc := func(io.Writer) error { return errors.New("") }
testDuration := time.Second * 5
tests := []struct {
profileType pb.ProfileType
duration *time.Duration
startCPUProfileFunc func(io.Writer) error
writeHeapProfileFunc func(io.Writer) error
wantBytes []byte
}{
{
profileType: pb.ProfileType_CPU,
duration: &testDuration,
startCPUProfileFunc: func(w io.Writer) error {
w.Write([]byte{1})
return nil
},
writeHeapProfileFunc: errFunc,
wantBytes: []byte{1},
},
{
profileType: pb.ProfileType_CPU,
startCPUProfileFunc: errFunc,
writeHeapProfileFunc: errFunc,
},
{
profileType: pb.ProfileType_CPU,
duration: &testDuration,
startCPUProfileFunc: func(w io.Writer) error {
w.Write([]byte{2})
return nil
},
writeHeapProfileFunc: func(w io.Writer) error {
w.Write([]byte{3})
return nil
},
wantBytes: []byte{2},
},
{
profileType: pb.ProfileType_HEAP,
startCPUProfileFunc: errFunc,
writeHeapProfileFunc: func(w io.Writer) error {
w.Write([]byte{4})
return nil
},
wantBytes: []byte{4},
},
{
profileType: pb.ProfileType_HEAP,
startCPUProfileFunc: errFunc,
writeHeapProfileFunc: errFunc,
},
{
profileType: pb.ProfileType_HEAP,
startCPUProfileFunc: func(w io.Writer) error {
w.Write([]byte{5})
return nil
},
writeHeapProfileFunc: func(w io.Writer) error {
w.Write([]byte{6})
return nil
},
wantBytes: []byte{6},
},
{
profileType: pb.ProfileType_PROFILE_TYPE_UNSPECIFIED,
startCPUProfileFunc: func(w io.Writer) error {
w.Write([]byte{7})
return nil
},
writeHeapProfileFunc: func(w io.Writer) error {
w.Write([]byte{8})
return nil
},
},
}
for _, tt := range tests {
mpc := mocks.NewMockProfilerServiceClient(ctrl)
a := createTestAgent(mpc)
startCPUProfile = tt.startCPUProfileFunc
stopCPUProfile = func() {}
writeHeapProfile = tt.writeHeapProfileFunc
var gotSleep *time.Duration
sleep = func(ctx context.Context, d time.Duration) error {
gotSleep = &d
return nil
}
p := &pb.Profile{ProfileType: tt.profileType}
if tt.duration != nil {
p.Duration = ptypes.DurationProto(*tt.duration)
}
if tt.wantBytes != nil {
wantProfile := &pb.Profile{
ProfileType: p.ProfileType,
Duration: p.Duration,
ProfileBytes: tt.wantBytes,
Labels: a.profileLabels,
}
wantRequest := pb.UpdateProfileRequest{
Profile: wantProfile,
}
mpc.EXPECT().UpdateProfile(ctx, gomock.Eq(&wantRequest)).Times(1)
} else {
mpc.EXPECT().UpdateProfile(gomock.Any(), gomock.Any()).MaxTimes(0)
}
a.profileAndUpload(ctx, p)
if tt.duration == nil {
if gotSleep != nil {
t.Errorf("profileAndUpload(%v) slept for: %v, want no sleep", p, gotSleep)
}
} else {
if gotSleep == nil {
t.Errorf("profileAndUpload(%v) didn't sleep, want sleep for: %v", p, tt.duration)
} else if *gotSleep != *tt.duration {
t.Errorf("profileAndUpload(%v) slept for wrong duration, got: %v, want: %v", p, gotSleep, tt.duration)
}
}
}
}
func TestRetry(t *testing.T) {
normalDuration := time.Second * 3
negativeDuration := time.Second * -3
tests := []struct {
trailers map[string]string
wantPause *time.Duration
}{
{
createTrailers(normalDuration),
&normalDuration,
},
{
createTrailers(negativeDuration),
nil,
},
{
map[string]string{retryInfoMetadata: "wrong format"},
nil,
},
{
map[string]string{},
nil,
},
}
for _, tt := range tests {
md := grpcmd.New(tt.trailers)
r := &retryer{
backoff: gax.Backoff{
Initial: initialBackoff,
Max: maxBackoff,
Multiplier: backoffMultiplier,
},
md: md,
}
pause, shouldRetry := r.Retry(status.Error(codes.Aborted, ""))
if !shouldRetry {
t.Error("retryer.Retry() returned shouldRetry false, want true")
}
if tt.wantPause != nil {
if pause != *tt.wantPause {
t.Errorf("retryer.Retry() returned wrong pause, got: %v, want: %v", pause, tt.wantPause)
}
} else {
if pause > initialBackoff {
t.Errorf("retryer.Retry() returned wrong pause, got: %v, want: < %v", pause, initialBackoff)
}
}
}
md := grpcmd.New(map[string]string{})
r := &retryer{
backoff: gax.Backoff{
Initial: initialBackoff,
Max: maxBackoff,
Multiplier: backoffMultiplier,
},
md: md,
}
for i := 0; i < 100; i++ {
pause, shouldRetry := r.Retry(errors.New(""))
if !shouldRetry {
t.Errorf("retryer.Retry() called %v times, returned shouldRetry false, want true", i)
}
if pause > maxBackoff {
t.Errorf("retryer.Retry() called %v times, returned wrong pause, got: %v, want: < %v", i, pause, maxBackoff)
}
}
}
func TestWithXGoogHeader(t *testing.T) {
ctx := withXGoogHeader(context.Background())
md, _ := grpcmd.FromOutgoingContext(ctx)
if xg := md[xGoogAPIMetadata]; len(xg) == 0 {
t.Errorf("withXGoogHeader() sets empty xGoogHeader")
} else {
if !strings.Contains(xg[0], "gl-go/") {
t.Errorf("withXGoogHeader() got: %v, want gl-go key", xg[0])
}
if !strings.Contains(xg[0], "gccl/") {
t.Errorf("withXGoogHeader() got: %v, want gccl key", xg[0])
}
if !strings.Contains(xg[0], "gax/") {
t.Errorf("withXGoogHeader() got: %v, want gax key", xg[0])
}
if !strings.Contains(xg[0], "grpc/") {
t.Errorf("withXGoogHeader() got: %v, want grpc key", xg[0])
}
}
}
func TestInitializeAgent(t *testing.T) {
oldConfig, oldMutexEnabled := config, mutexEnabled
defer func() {
config, mutexEnabled = oldConfig, oldMutexEnabled
}()
for _, tt := range []struct {
config Config
enableMutex bool
wantProfileTypes []pb.ProfileType
wantDeploymentLabels map[string]string
wantProfileLabels map[string]string
}{
{
config: Config{ServiceVersion: testSvcVersion, zone: testZone},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
wantDeploymentLabels: map[string]string{zoneNameLabel: testZone, versionLabel: testSvcVersion},
wantProfileLabels: map[string]string{},
},
{
config: Config{zone: testZone},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
wantDeploymentLabels: map[string]string{zoneNameLabel: testZone},
wantProfileLabels: map[string]string{},
},
{
config: Config{ServiceVersion: testSvcVersion},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
wantDeploymentLabels: map[string]string{versionLabel: testSvcVersion},
wantProfileLabels: map[string]string{},
},
{
config: Config{instance: testInstance},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS},
wantDeploymentLabels: map[string]string{},
wantProfileLabels: map[string]string{instanceLabel: testInstance},
},
{
config: Config{instance: testInstance},
enableMutex: true,
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_HEAP, pb.ProfileType_THREADS, pb.ProfileType_CONTENTION},
wantDeploymentLabels: map[string]string{},
wantProfileLabels: map[string]string{instanceLabel: testInstance},
},
{
config: Config{NoHeapProfiling: true},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU, pb.ProfileType_THREADS},
wantDeploymentLabels: map[string]string{},
wantProfileLabels: map[string]string{},
},
{
config: Config{NoHeapProfiling: true, NoGoroutineProfiling: true},
wantProfileTypes: []pb.ProfileType{pb.ProfileType_CPU},
wantDeploymentLabels: map[string]string{},
wantProfileLabels: map[string]string{},
},
} {
config = tt.config
config.ProjectID = testProjectID
config.Target = testTarget
mutexEnabled = tt.enableMutex
a := initializeAgent(nil)
wantDeployment := &pb.Deployment{
ProjectId: testProjectID,
Target: testTarget,
Labels: tt.wantDeploymentLabels,
}
if !testutil.Equal(a.deployment, wantDeployment) {
t.Errorf("initializeAgent() got deployment: %v, want %v", a.deployment, wantDeployment)
}
if !testutil.Equal(a.profileLabels, tt.wantProfileLabels) {
t.Errorf("initializeAgent() got profile labels: %v, want %v", a.profileLabels, tt.wantProfileLabels)
}
if !testutil.Equal(a.profileTypes, tt.wantProfileTypes) {
t.Errorf("initializeAgent() got profile types: %v, want %v", a.profileTypes, tt.wantProfileTypes)
}
}
}
func TestInitializeConfig(t *testing.T) {
oldConfig, oldService, oldVersion, oldEnvProjectID, oldGetProjectID, oldGetInstanceName, oldGetZone, oldOnGCE := config, os.Getenv("GAE_SERVICE"), os.Getenv("GAE_VERSION"), os.Getenv("GOOGLE_CLOUD_PROJECT"), getProjectID, getInstanceName, getZone, onGCE
defer func() {
config, getProjectID, getInstanceName, getZone, onGCE = oldConfig, oldGetProjectID, oldGetInstanceName, oldGetZone, oldOnGCE
if err := os.Setenv("GAE_SERVICE", oldService); err != nil {
t.Fatal(err)
}
if err := os.Setenv("GAE_VERSION", oldVersion); err != nil {
t.Fatal(err)
}
if err := os.Setenv("GOOGLE_CLOUD_PROJECT", oldEnvProjectID); err != nil {
t.Fatal(err)
}
}()
const (
testGAEService = "test-gae-service"
testGAEVersion = "test-gae-version"
testGCEProjectID = "test-gce-project-id"
testEnvProjectID = "test-env-project-id"
)
for _, tt := range []struct {
desc string
config Config
wantConfig Config
wantErrorString string
onGAE bool
onGCE bool
envProjectID bool
}{
{
"accepts service name",
Config{Service: testService},
Config{Target: testService, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
false,
true,
false,
},
{
"accepts target name",
Config{Target: testTarget},
Config{Target: testTarget, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
false,
true,
false,
},
{
"env project overrides GCE project",
Config{Service: testService},
Config{Target: testService, ProjectID: testEnvProjectID, zone: testZone, instance: testInstance},
"",
false,
true,
true,
},
{
"requires service name",
Config{},
Config{},
"service name must be specified in the configuration",
false,
true,
false,
},
{
"accepts service name from config and service version from GAE",
Config{Service: testService},
Config{Target: testService, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
true,
true,
false,
},
{
"accepts target name from config and service version from GAE",
Config{Target: testTarget},
Config{Target: testTarget, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
true,
true,
false,
},
{
"reads both service name and version from GAE env vars",
Config{},
Config{Target: testGAEService, ServiceVersion: testGAEVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
true,
true,
false,
},
{
"accepts service version from config",
Config{Service: testService, ServiceVersion: testSvcVersion},
Config{Target: testService, ServiceVersion: testSvcVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
false,
true,
false,
},
{
"configured version has priority over GAE-provided version",
Config{Service: testService, ServiceVersion: testSvcVersion},
Config{Target: testService, ServiceVersion: testSvcVersion, ProjectID: testGCEProjectID, zone: testZone, instance: testInstance},
"",
true,
true,
false,
},
{
"configured project ID has priority over metadata-provided project ID",
Config{Service: testService, ProjectID: testProjectID},
Config{Target: testService, ProjectID: testProjectID, zone: testZone, instance: testInstance},
"",
false,
true,
false,
},
{
"configured project ID has priority over environment project ID",
Config{Service: testService, ProjectID: testProjectID},
Config{Target: testService, ProjectID: testProjectID},
"",
false,
false,
true,
},
{
"requires project ID if not on GCE",
Config{Service: testService},
Config{Target: testService},
"project ID must be specified in the configuration if running outside of GCP",
false,
false,
false,
},
} {
t.Logf("Running test: %s", tt.desc)
envService, envVersion := "", ""
if tt.onGAE {
envService, envVersion = testGAEService, testGAEVersion
}
if err := os.Setenv("GAE_SERVICE", envService); err != nil {
t.Fatal(err)
}
if err := os.Setenv("GAE_VERSION", envVersion); err != nil {
t.Fatal(err)
}
if tt.onGCE {
onGCE = func() bool { return true }
getProjectID = func() (string, error) { return testGCEProjectID, nil }
getZone = func() (string, error) { return testZone, nil }
getInstanceName = func() (string, error) { return testInstance, nil }
} else {
onGCE = func() bool { return false }
getProjectID = func() (string, error) { return "", fmt.Errorf("test get project id error") }
getZone = func() (string, error) { return "", fmt.Errorf("test get zone error") }
getInstanceName = func() (string, error) { return "", fmt.Errorf("test get instance error") }
}
envProjectID := ""
if tt.envProjectID {
envProjectID = testEnvProjectID
}
if err := os.Setenv("GOOGLE_CLOUD_PROJECT", envProjectID); err != nil {
t.Fatal(err)
}
errorString := ""
if err := initializeConfig(tt.config); err != nil {
errorString = err.Error()
}
if !strings.Contains(errorString, tt.wantErrorString) {
t.Errorf("initializeConfig(%v) got error: %v, want contain %v", tt.config, errorString, tt.wantErrorString)
}
if tt.wantErrorString == "" {
tt.wantConfig.APIAddr = apiAddress
}
tt.wantConfig.Service = tt.config.Service
if config != tt.wantConfig {
t.Errorf("initializeConfig(%v) got: %v, want %v", tt.config, config, tt.wantConfig)
}
}
for _, tt := range []struct {
wantErrorString string
getProjectIDError bool
getZoneError bool
getInstanceError bool
}{
{
wantErrorString: "failed to get the project ID from Compute Engine:",
getProjectIDError: true,
},
{
wantErrorString: "failed to get zone from Compute Engine:",
getZoneError: true,
},
{
wantErrorString: "failed to get instance from Compute Engine:",
getInstanceError: true,
},
} {
onGCE = func() bool { return true }
if tt.getProjectIDError {
getProjectID = func() (string, error) { return "", fmt.Errorf("test get project ID error") }
} else {
getProjectID = func() (string, error) { return testGCEProjectID, nil }
}
if tt.getZoneError {
getZone = func() (string, error) { return "", fmt.Errorf("test get zone error") }
} else {
getZone = func() (string, error) { return testZone, nil }
}
if tt.getInstanceError {
getInstanceName = func() (string, error) { return "", fmt.Errorf("test get instance error") }
} else {
getInstanceName = func() (string, error) { return testInstance, nil }
}
errorString := ""
if err := initializeConfig(Config{Service: testService}); err != nil {
errorString = err.Error()
}
if !strings.Contains(errorString, tt.wantErrorString) {
t.Errorf("initializeConfig() got error: %v, want contain %v", errorString, tt.wantErrorString)
}
}
}
type fakeProfilerServer struct {
pb.ProfilerServiceServer
count int
gotProfiles map[string][]byte
done chan bool
}
func (fs *fakeProfilerServer) CreateProfile(ctx context.Context, in *pb.CreateProfileRequest) (*pb.Profile, error) {
fs.count++
switch fs.count {
case 1:
return &pb.Profile{Name: "testCPU", ProfileType: pb.ProfileType_CPU, Duration: ptypes.DurationProto(testProfileDuration)}, nil
case 2:
return &pb.Profile{Name: "testHeap", ProfileType: pb.ProfileType_HEAP}, nil
default:
select {}
}
}
func (fs *fakeProfilerServer) UpdateProfile(ctx context.Context, in *pb.UpdateProfileRequest) (*pb.Profile, error) {
switch in.Profile.ProfileType {
case pb.ProfileType_CPU:
fs.gotProfiles["CPU"] = in.Profile.ProfileBytes
case pb.ProfileType_HEAP:
fs.gotProfiles["HEAP"] = in.Profile.ProfileBytes
fs.done <- true
}
return in.Profile, nil
}
func profileeLoop(quit chan bool) {
for {
select {
case <-quit:
return
default:
profileeWork()
}
}
}
func profileeWork() {
data := make([]byte, 1024*1024)
rand.Read(data)
var b bytes.Buffer
gz := gzip.NewWriter(&b)
if _, err := gz.Write(data); err != nil {
log.Println("failed to write to gzip stream", err)
return
}
if err := gz.Flush(); err != nil {
log.Println("failed to flush to gzip stream", err)
return
}
if err := gz.Close(); err != nil {
log.Println("failed to close gzip stream", err)
}
}
func validateProfile(rawData []byte, wantFunctionName string) error {
p, err := profile.ParseData(rawData)
if err != nil {
return fmt.Errorf("ParseData failed: %v", err)
}
if len(p.Sample) == 0 {
return fmt.Errorf("profile contains zero samples: %v", p)
}
if len(p.Location) == 0 {
return fmt.Errorf("profile contains zero locations: %v", p)
}
if len(p.Function) == 0 {
return fmt.Errorf("profile contains zero functions: %v", p)
}
for _, l := range p.Location {
if len(l.Line) > 0 && l.Line[0].Function != nil && strings.Contains(l.Line[0].Function.Name, wantFunctionName) {
return nil
}
}
return fmt.Errorf("wanted function name %s not found in the profile", wantFunctionName)
}
func TestDeltaMutexProfile(t *testing.T) {
oldMutexEnabled, oldMaxProcs := mutexEnabled, runtime.GOMAXPROCS(10)
defer func() {
mutexEnabled = oldMutexEnabled
runtime.GOMAXPROCS(oldMaxProcs)
}()
if mutexEnabled = enableMutexProfiling(); !mutexEnabled {
t.Skip("Go too old - mutex profiling not supported.")
}
hog(time.Second, mutexHog)
go func() {
hog(2*time.Second, backgroundHog)
}()
var prof bytes.Buffer
if err := deltaMutexProfile(context.Background(), time.Second, &prof); err != nil {
t.Fatalf("deltaMutexProfile() got error: %v", err)
}
p, err := profile.Parse(&prof)
if err != nil {
t.Fatalf("profile.Parse() got error: %v", err)
}
if s := sum(p, "mutexHog"); s != 0 {
t.Errorf("mutexHog found in the delta mutex profile (sum=%d):\n%s", s, p)
}
if s := sum(p, "backgroundHog"); s <= 0 {
t.Errorf("backgroundHog not in the delta mutex profile (sum=%d):\n%s", s, p)
}
}
// sum returns the sum of all mutex counts from the samples whose
// stacks include the specified function name.
func sum(p *profile.Profile, fname string) int64 {
locIDs := map[*profile.Location]bool{}
for _, loc := range p.Location {
for _, l := range loc.Line {
if strings.Contains(l.Function.Name, fname) {
locIDs[loc] = true
break
}
}
}
var s int64
for _, sample := range p.Sample {
for _, loc := range sample.Location {
if locIDs[loc] {
s += sample.Value[0]
break
}
}
}
return s
}
func mutexHog(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration) {
for time.Since(start) < dt {
mu1.Lock()
runtime.Gosched()
mu2.Lock()
mu1.Unlock()
mu2.Unlock()
}
}
// backgroundHog is identical to mutexHog. We keep them separate
// in order to distinguish them with function names in the stack trace.
func backgroundHog(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration) {
for time.Since(start) < dt {
mu1.Lock()
runtime.Gosched()
mu2.Lock()
mu1.Unlock()
mu2.Unlock()
}
}
func hog(dt time.Duration, hogger func(mu1, mu2 *sync.Mutex, start time.Time, dt time.Duration)) {
start := time.Now()
mu1 := new(sync.Mutex)
mu2 := new(sync.Mutex)
var wg sync.WaitGroup
wg.Add(10)
for i := 0; i < 10; i++ {
go func() {
defer wg.Done()
hogger(mu1, mu2, start, dt)
}()
}
wg.Wait()
}
func TestAgentWithServer(t *testing.T) {
oldDialGRPC, oldConfig := dialGRPC, config
defer func() {
dialGRPC, config = oldDialGRPC, oldConfig
}()
srv, err := testutil.NewServer()
if err != nil {
t.Fatalf("testutil.NewServer(): %v", err)
}
fakeServer := &fakeProfilerServer{gotProfiles: map[string][]byte{}, done: make(chan bool)}
pb.RegisterProfilerServiceServer(srv.Gsrv, fakeServer)
srv.Start()
dialGRPC = gtransport.DialInsecure
if err := Start(Config{
Target: testTarget,
ProjectID: testProjectID,
APIAddr: srv.Addr,
instance: testInstance,
zone: testZone,
}); err != nil {
t.Fatalf("Start(): %v", err)
}
quitProfilee := make(chan bool)
go profileeLoop(quitProfilee)
select {
case <-fakeServer.done:
case <-time.After(testServerTimeout):
t.Errorf("got timeout after %v, want fake server done", testServerTimeout)
}
quitProfilee <- true
for _, pType := range []string{"CPU", "HEAP"} {
if profile, ok := fakeServer.gotProfiles[pType]; !ok {
t.Errorf("fakeServer.gotProfiles[%s] got no profile, want profile", pType)
} else if err := validateProfile(profile, "profilee"); err != nil {
t.Errorf("validateProfile(%s) got error: %v", pType, err)
}
}
}
+501
View File
@@ -0,0 +1,501 @@
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package proftest contains test helpers for profiler agent integration tests.
// This package is experimental.
// golang.org/x/build/kubernetes/dialer.go imports "context" package (rather
// than "golang.org/x/net/context") and that does not exist in Go 1.6 or
// earlier.
// +build go1.7
package proftest
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"cloud.google.com/go/storage"
"golang.org/x/build/kubernetes"
k8sapi "golang.org/x/build/kubernetes/api"
"golang.org/x/build/kubernetes/gke"
"golang.org/x/net/context"
cloudbuild "google.golang.org/api/cloudbuild/v1"
compute "google.golang.org/api/compute/v1"
container "google.golang.org/api/container/v1"
"google.golang.org/api/googleapi"
)
const (
monitorWriteScope = "https://www.googleapis.com/auth/monitoring.write"
storageReadScope = "https://www.googleapis.com/auth/devstorage.read_only"
)
// TestRunner has common elements used for testing profiling agents on a range
// of environments.
type TestRunner struct {
Client *http.Client
}
// GCETestRunner supports testing a profiling agent on GCE.
type GCETestRunner struct {
TestRunner
ComputeService *compute.Service
}
// GKETestRunner supports testing a profiling agent on GKE.
type GKETestRunner struct {
TestRunner
ContainerService *container.Service
StorageClient *storage.Client
Dockerfile string
}
// ProfileResponse contains the response produced when querying profile server.
type ProfileResponse struct {
Profile ProfileData `json:"profile"`
NumProfiles int32 `json:"numProfiles"`
Deployments []interface{} `json:"deployments"`
}
// ProfileData has data of a single profile.
type ProfileData struct {
Samples []int32 `json:"samples"`
SampleMetrics interface{} `json:"sampleMetrics"`
DefaultMetricType string `json:"defaultMetricType"`
TreeNodes interface{} `json:"treeNodes"`
Functions functionArray `json:"functions"`
SourceFiles interface{} `json:"sourceFiles"`
}
type functionArray struct {
Name []string `json:"name"`
Sourcefile []int32 `json:"sourceFile"`
}
// InstanceConfig is configuration for starting single GCE instance for
// profiling agent test case.
type InstanceConfig struct {
ProjectID string
Zone string
Name string
StartupScript string
MachineType string
}
// ClusterConfig is configuration for starting single GKE cluster for profiling
// agent test case.
type ClusterConfig struct {
ProjectID string
Zone string
ClusterName string
PodName string
ImageSourceName string
ImageName string
Bucket string
Dockerfile string
}
// HasFunction returns nil if the function is present, or, if the function is
// not present, and error providing more details why the function is not
// present.
func (pr *ProfileResponse) HasFunction(functionName string) error {
if pr.NumProfiles == 0 {
return fmt.Errorf("failed to find function name %s in profile: profile response contains zero profiles: %v", functionName, pr)
}
if len(pr.Deployments) == 0 {
return fmt.Errorf("failed to find function name %s in profile: profile response contains zero deployments: %v", functionName, pr)
}
if len(pr.Profile.Functions.Name) == 0 {
return fmt.Errorf("failed to find function name %s in profile: profile does not have function data", functionName)
}
for _, name := range pr.Profile.Functions.Name {
if strings.Contains(name, functionName) {
return nil
}
}
return fmt.Errorf("failed to find function name %s in profile", functionName)
}
// StartInstance starts a GCE Instance with name, zone, and projectId specified
// by the inst, and which runs the startup script specified in inst.
func (tr *GCETestRunner) StartInstance(ctx context.Context, inst *InstanceConfig) error {
img, err := tr.ComputeService.Images.GetFromFamily("debian-cloud", "debian-9").Context(ctx).Do()
if err != nil {
return err
}
_, err = tr.ComputeService.Instances.Insert(inst.ProjectID, inst.Zone, &compute.Instance{
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", inst.Zone, inst.MachineType),
Name: inst.Name,
Disks: []*compute.AttachedDisk{{
AutoDelete: true, // delete the disk when the VM is deleted.
Boot: true,
Type: "PERSISTENT",
Mode: "READ_WRITE",
InitializeParams: &compute.AttachedDiskInitializeParams{
SourceImage: img.SelfLink,
DiskType: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/pd-standard", inst.ProjectID, inst.Zone),
},
}},
NetworkInterfaces: []*compute.NetworkInterface{{
Network: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/default", inst.ProjectID),
AccessConfigs: []*compute.AccessConfig{{
Name: "External NAT",
}},
}},
Metadata: &compute.Metadata{
Items: []*compute.MetadataItems{{
Key: "startup-script",
Value: googleapi.String(inst.StartupScript),
}},
},
ServiceAccounts: []*compute.ServiceAccount{{
Email: "default",
Scopes: []string{
monitorWriteScope,
},
}},
}).Do()
return err
}
// DeleteInstance deletes an instance with project id, name, and zone matched
// by inst.
func (tr *GCETestRunner) DeleteInstance(ctx context.Context, inst *InstanceConfig) error {
if _, err := tr.ComputeService.Instances.Delete(inst.ProjectID, inst.Zone, inst.Name).Context(ctx).Do(); err != nil {
return fmt.Errorf("Instances.Delete(%s) got error: %v", inst.Name, err)
}
return nil
}
// PollForSerialOutput polls serial port 2 of the GCE instance specified by
// inst and returns when the finishString appears in the serial output
// of the instance, or when the context times out.
func (tr *GCETestRunner) PollForSerialOutput(ctx context.Context, inst *InstanceConfig, finishString string) error {
var output string
defer func() {
log.Printf("Serial port output for %s:\n%s", inst.Name, output)
}()
for {
select {
case <-ctx.Done():
case <-time.After(20 * time.Second):
resp, err := tr.ComputeService.Instances.GetSerialPortOutput(inst.ProjectID, inst.Zone, inst.Name).Port(2).Context(ctx).Do()
if err != nil {
// Transient failure.
log.Printf("Transient error getting serial port output from instance %s (will retry): %v", inst.Name, err)
continue
}
if output = resp.Contents; strings.Contains(output, finishString) {
return nil
}
}
}
}
// QueryProfiles retrieves profiles of a specific type, from a specific time
// range, associated with a particular service and project.
func (tr *TestRunner) QueryProfiles(projectID, service, startTime, endTime, profileType string) (ProfileResponse, error) {
queryURL := fmt.Sprintf("https://cloudprofiler.googleapis.com/v2/projects/%s/profiles:query", projectID)
const queryJSONFmt = `{"endTime": "%s", "profileType": "%s","startTime": "%s", "target": "%s"}`
queryRequest := fmt.Sprintf(queryJSONFmt, endTime, profileType, startTime, service)
resp, err := tr.Client.Post(queryURL, "application/json", strings.NewReader(queryRequest))
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to query API: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to read response body: %v", err)
}
var pr ProfileResponse
if err := json.Unmarshal(body, &pr); err != nil {
return ProfileResponse{}, err
}
return pr, nil
}
// createAndPublishDockerImage creates a docker image from source code in a GCS
// bucket and pushes the image to Google Container Registry.
func (tr *GKETestRunner) createAndPublishDockerImage(ctx context.Context, projectID, sourceBucket, sourceObject, ImageName string) error {
cloudbuildService, err := cloudbuild.New(tr.Client)
build := &cloudbuild.Build{
Source: &cloudbuild.Source{
StorageSource: &cloudbuild.StorageSource{
Bucket: sourceBucket,
Object: sourceObject,
},
},
Steps: []*cloudbuild.BuildStep{
{
Name: "gcr.io/cloud-builders/docker",
Args: []string{"build", "-t", ImageName, "."},
},
},
Images: []string{ImageName},
}
op, err := cloudbuildService.Projects.Builds.Create(projectID, build).Context(ctx).Do()
if err != nil {
return fmt.Errorf("failed to create image: %v", err)
}
opID := op.Name
// Wait for creating image.
for {
select {
case <-ctx.Done():
return fmt.Errorf("timed out waiting creating image")
case <-time.After(10 * time.Second):
op, err := cloudbuildService.Operations.Get(opID).Context(ctx).Do()
if err != nil {
log.Printf("Transient error getting operation (will retry): %v", err)
break
}
if op.Done == true {
log.Printf("Published image %s to Google Container Registry.", ImageName)
return nil
}
}
}
}
type imageResponse struct {
Manifest map[string]interface{} `json:"manifest"`
Name string `json:"name"`
Tags []string `json:"tags"`
}
// deleteDockerImage deletes a docker image from Google Container Registry.
func (tr *GKETestRunner) deleteDockerImage(ctx context.Context, ImageName string) []error {
queryImageURL := fmt.Sprintf("https://gcr.io/v2/%s/tags/list", ImageName)
resp, err := tr.Client.Get(queryImageURL)
if err != nil {
return []error{fmt.Errorf("failed to list tags: %v", err)}
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return []error{err}
}
var ir imageResponse
if err := json.Unmarshal(body, &ir); err != nil {
return []error{err}
}
const deleteImageURLFmt = "https://gcr.io/v2/%s/manifests/%s"
var errs []error
for _, tag := range ir.Tags {
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, tag)); err != nil {
errs = append(errs, fmt.Errorf("failed to delete tag %s: %v", tag, err))
}
}
for manifest := range ir.Manifest {
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, manifest)); err != nil {
errs = append(errs, fmt.Errorf("failed to delete manifest %s: %v", manifest, err))
}
}
return errs
}
func deleteDockerImageResource(client *http.Client, url string) error {
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return fmt.Errorf("failed to get request: %v", err)
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to delete resource: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to delete resource: status code = %d", resp.StatusCode)
}
return nil
}
func (tr *GKETestRunner) createCluster(ctx context.Context, client *http.Client, projectID, zone, ClusterName string) error {
request := &container.CreateClusterRequest{Cluster: &container.Cluster{
Name: ClusterName,
InitialNodeCount: 3,
NodeConfig: &container.NodeConfig{
OauthScopes: []string{
storageReadScope,
},
},
}}
op, err := tr.ContainerService.Projects.Zones.Clusters.Create(projectID, zone, request).Context(ctx).Do()
if err != nil {
return fmt.Errorf("failed to create cluster %s: %v", ClusterName, err)
}
opID := op.Name
// Wait for creating cluster.
for {
select {
case <-ctx.Done():
return fmt.Errorf("timed out waiting creating cluster")
case <-time.After(10 * time.Second):
op, err := tr.ContainerService.Projects.Zones.Operations.Get(projectID, zone, opID).Context(ctx).Do()
if err != nil {
log.Printf("Transient error getting operation (will retry): %v", err)
break
}
if op.Status == "DONE" {
log.Printf("Created cluster %s.", ClusterName)
return nil
}
if op.Status == "ABORTING" {
return fmt.Errorf("create cluster operation is aborted")
}
}
}
}
func (tr *GKETestRunner) deployContainer(ctx context.Context, kubernetesClient *kubernetes.Client, podName, ImageName string) error {
pod := &k8sapi.Pod{
ObjectMeta: k8sapi.ObjectMeta{
Name: podName,
},
Spec: k8sapi.PodSpec{
Containers: []k8sapi.Container{
{
Name: "profiler-test",
Image: fmt.Sprintf("gcr.io/%s:latest", ImageName),
},
},
},
}
if _, err := kubernetesClient.RunLongLivedPod(ctx, pod); err != nil {
return fmt.Errorf("failed to run pod %s: %v", podName, err)
}
return nil
}
// PollPodLog polls the log of the kubernetes client and returns when the
// finishString appears in the log, or when the context times out.
func (tr *GKETestRunner) PollPodLog(ctx context.Context, kubernetesClient *kubernetes.Client, podName, finishString string) error {
var output string
defer func() {
log.Printf("Log for pod %s:\n%s", podName, output)
}()
for {
select {
case <-ctx.Done():
return fmt.Errorf("timed out waiting profiling finishing on container")
case <-time.After(20 * time.Second):
var err error
output, err = kubernetesClient.PodLog(ctx, podName)
if err != nil {
// Transient failure.
log.Printf("Transient error getting log (will retry): %v", err)
continue
}
if strings.Contains(output, finishString) {
return nil
}
}
}
}
// DeleteClusterAndImage deletes cluster and images used to create cluster.
func (tr *GKETestRunner) DeleteClusterAndImage(ctx context.Context, cfg *ClusterConfig) []error {
var errs []error
if err := tr.StorageClient.Bucket(cfg.Bucket).Object(cfg.ImageSourceName).Delete(ctx); err != nil {
errs = append(errs, fmt.Errorf("failed to delete storage client: %v", err))
}
for _, err := range tr.deleteDockerImage(ctx, cfg.ImageName) {
errs = append(errs, fmt.Errorf("failed to delete docker image: %v", err))
}
if _, err := tr.ContainerService.Projects.Zones.Clusters.Delete(cfg.ProjectID, cfg.Zone, cfg.ClusterName).Context(ctx).Do(); err != nil {
errs = append(errs, fmt.Errorf("failed to delete cluster %s: %v", cfg.ClusterName, err))
}
return errs
}
// StartAndDeployCluster creates image needed for cluster, then starts and
// deploys to cluster.
func (tr *GKETestRunner) StartAndDeployCluster(ctx context.Context, cfg *ClusterConfig) error {
if err := tr.uploadImageSource(ctx, cfg.Bucket, cfg.ImageSourceName, cfg.Dockerfile); err != nil {
return fmt.Errorf("failed to upload image source: %v", err)
}
createImageCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
if err := tr.createAndPublishDockerImage(createImageCtx, cfg.ProjectID, cfg.Bucket, cfg.ImageSourceName, fmt.Sprintf("gcr.io/%s", cfg.ImageName)); err != nil {
return fmt.Errorf("failed to create and publish docker image %s: %v", cfg.ImageName, err)
}
kubernetesClient, err := gke.NewClient(ctx, cfg.ClusterName, gke.OptZone(cfg.Zone), gke.OptProject(cfg.ProjectID))
if err != nil {
return fmt.Errorf("failed to create new GKE client: %v", err)
}
deployContainerCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
if err := tr.deployContainer(deployContainerCtx, kubernetesClient, cfg.PodName, cfg.ImageName); err != nil {
return fmt.Errorf("failed to deploy image %q to pod %q: %v", cfg.PodName, cfg.ImageName, err)
}
return nil
}
// uploadImageSource uploads source code for building docker image to GCS.
func (tr *GKETestRunner) uploadImageSource(ctx context.Context, bucket, objectName, dockerfile string) error {
zipBuf := new(bytes.Buffer)
z := zip.NewWriter(zipBuf)
f, err := z.Create("Dockerfile")
if err != nil {
return err
}
if _, err := f.Write([]byte(dockerfile)); err != nil {
return err
}
if err := z.Close(); err != nil {
return err
}
wc := tr.StorageClient.Bucket(bucket).Object(objectName).NewWriter(ctx)
wc.ContentType = "application/zip"
wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
if _, err := wc.Write(zipBuf.Bytes()); err != nil {
return err
}
return wc.Close()
}
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package profiler
import (
"bytes"
"regexp"
"runtime"
"strings"
"github.com/google/pprof/profile"
)
var shouldAssumeSymbolized = isSymbolizedGoVersion(runtime.Version())
type function interface {
Name() string
FileLine(pc uintptr) (string, int)
}
// funcForPC is a wrapper for runtime.FuncForPC. Defined as var for testing.
var funcForPC = func(pc uintptr) function {
if f := runtime.FuncForPC(pc); f != nil {
return f
}
return nil
}
// parseAndSymbolize parses a profile from a buffer, symbolizes it
// if it's not yet symbolized, and writes the profile back as a
// gzip-compressed marshaled protobuf.
func parseAndSymbolize(data *bytes.Buffer) error {
p, err := profile.ParseData(data.Bytes())
if err != nil {
return err
}
// Do nothing if the profile is already symbolized.
if symbolized(p) {
return nil
}
// Clear the profile functions to avoid creating duplicates.
p.Function = nil
symbolize(p)
data.Reset()
return p.Write(data)
}
// isSymbolizedGoVersion returns true if Go version equals to or is
// higher than Go 1.9. Starting Go 1.9 the profiles are symbolized
// by runtime/pprof.
func isSymbolizedGoVersion(goVersion string) bool {
r, err := regexp.Compile(`go(1\.9|1\.[1-9][0-9]|[2-9]).*`)
if err == nil && r.MatchString(goVersion) {
return true
}
return false
}
// symbolized checks if all locations have symbolized function
// information.
func symbolized(p *profile.Profile) bool {
for _, l := range p.Location {
if len(l.Line) == 0 || l.Line[0].Function == nil {
return false
}
}
return true
}
func symbolize(p *profile.Profile) {
fns := profileFunctionMap{}
for _, l := range p.Location {
pc := uintptr(l.Address)
f := funcForPC(pc)
if f == nil {
continue
}
file, lineno := f.FileLine(pc)
l.Line = []profile.Line{
{
Function: fns.findOrAddFunction(f.Name(), file, p),
Line: int64(lineno),
},
}
}
// Trim runtime functions. Always hide runtime.goexit. Other runtime
// functions are only hidden for heap profile when they appear at the beginning.
isHeapProfile := p.PeriodType != nil && p.PeriodType.Type == "space"
for _, s := range p.Sample {
show := !isHeapProfile
var i int
for _, l := range s.Location {
if len(l.Line) > 0 && l.Line[0].Function != nil {
name := l.Line[0].Function.Name
if name == "runtime.goexit" || !show && strings.HasPrefix(name, "runtime.") {
continue
}
}
show = true
s.Location[i] = l
i++
}
// If all locations of a sample are trimmed, keep the root location.
if i == 0 && len(s.Location) > 0 {
s.Location[0] = s.Location[len(s.Location)-1]
i = 1
}
s.Location = s.Location[:i]
}
}
type profileFunctionMap map[profile.Function]*profile.Function
func (fns profileFunctionMap) findOrAddFunction(name, filename string, p *profile.Profile) *profile.Function {
f := profile.Function{
Name: name,
SystemName: name,
Filename: filename,
}
if fp := fns[f]; fp != nil {
return fp
}
fp := new(profile.Function)
fns[f] = fp
*fp = f
fp.ID = uint64(len(p.Function) + 1)
p.Function = append(p.Function, fp)
return fp
}
+229
View File
@@ -0,0 +1,229 @@
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package profiler
import (
"bytes"
"testing"
"cloud.google.com/go/internal/testutil"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/pprof/profile"
)
type fakeFunc struct {
name string
file string
lineno int
}
func (f *fakeFunc) Name() string {
return f.name
}
func (f *fakeFunc) FileLine(_ uintptr) (string, int) {
return f.file, f.lineno
}
var cmpOpt = cmpopts.IgnoreUnexported(profile.Profile{}, profile.Function{},
profile.Line{}, profile.Location{}, profile.Sample{}, profile.ValueType{})
// TestRuntimeFunctionTrimming tests if symbolize trims runtime functions as intended.
func TestRuntimeFunctionTrimming(t *testing.T) {
fakeFuncMap := map[uintptr]*fakeFunc{
0x10: &fakeFunc{"runtime.goexit", "runtime.go", 10},
0x20: &fakeFunc{"runtime.other", "runtime.go", 20},
0x30: &fakeFunc{"foo", "foo.go", 30},
0x40: &fakeFunc{"bar", "bar.go", 40},
}
backupFuncForPC := funcForPC
funcForPC = func(pc uintptr) function {
return fakeFuncMap[pc]
}
defer func() {
funcForPC = backupFuncForPC
}()
testLoc := []*profile.Location{
{ID: 1, Address: 0x10},
{ID: 2, Address: 0x20},
{ID: 3, Address: 0x30},
{ID: 4, Address: 0x40},
}
testProfile := &profile.Profile{
Sample: []*profile.Sample{
{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[2]}},
{Location: []*profile.Location{testLoc[1], testLoc[3], testLoc[2]}},
{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[1]}},
{Location: []*profile.Location{testLoc[3], testLoc[2], testLoc[0]}},
{Location: []*profile.Location{testLoc[0], testLoc[1], testLoc[3], testLoc[0]}},
{Location: []*profile.Location{testLoc[1], testLoc[0]}},
},
Location: testLoc,
}
testProfiles := make([]*profile.Profile, 2)
testProfiles[0] = testProfile.Copy()
testProfiles[1] = testProfile.Copy()
// Test case for CPU profile.
testProfiles[0].PeriodType = &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
// Test case for heap profile.
testProfiles[1].PeriodType = &profile.ValueType{Type: "space", Unit: "bytes"}
wantFunc := []*profile.Function{
{ID: 1, Name: "runtime.goexit", SystemName: "runtime.goexit", Filename: "runtime.go"},
{ID: 2, Name: "runtime.other", SystemName: "runtime.other", Filename: "runtime.go"},
{ID: 3, Name: "foo", SystemName: "foo", Filename: "foo.go"},
{ID: 4, Name: "bar", SystemName: "bar", Filename: "bar.go"},
}
wantLoc := []*profile.Location{
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
{ID: 3, Address: 0x30, Line: []profile.Line{{Function: wantFunc[2], Line: 30}}},
{ID: 4, Address: 0x40, Line: []profile.Line{{Function: wantFunc[3], Line: 40}}},
}
wantProfiles := []*profile.Profile{
{
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
Sample: []*profile.Sample{
{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[1], wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[1], wantLoc[3]}},
{Location: []*profile.Location{wantLoc[1]}},
},
Location: wantLoc,
Function: wantFunc,
},
{
PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},
Sample: []*profile.Sample{
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[3], wantLoc[2], wantLoc[1]}},
{Location: []*profile.Location{wantLoc[3], wantLoc[2]}},
{Location: []*profile.Location{wantLoc[3]}},
{Location: []*profile.Location{wantLoc[0]}},
},
Location: wantLoc,
Function: wantFunc,
},
}
for i := 0; i < 2; i++ {
symbolize(testProfiles[i])
if !testutil.Equal(testProfiles[i], wantProfiles[i], cmpOpt) {
t.Errorf("incorrect trimming (testcase = %d): got {%v}, want {%v}", i, testProfiles[i], wantProfiles[i])
}
}
}
// TestParseAndSymbolize tests if parseAndSymbolize parses and symbolizes
// profiles as intended.
func TestParseAndSymbolize(t *testing.T) {
fakeFuncMap := map[uintptr]*fakeFunc{
0x10: &fakeFunc{"foo", "foo.go", 10},
0x20: &fakeFunc{"bar", "bar.go", 20},
}
backupFuncForPC := funcForPC
funcForPC = func(pc uintptr) function {
return fakeFuncMap[pc]
}
defer func() {
funcForPC = backupFuncForPC
}()
testLoc := []*profile.Location{
{ID: 1, Address: 0x10},
{ID: 2, Address: 0x20},
}
testProfile := &profile.Profile{
SampleType: []*profile.ValueType{
&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
},
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
Sample: []*profile.Sample{
{Location: []*profile.Location{testLoc[0], testLoc[1]}, Value: []int64{1}},
{Location: []*profile.Location{testLoc[1]}, Value: []int64{1}},
},
Location: testLoc,
}
testProfiles := make([]*profile.Profile, 2)
testProfiles[0] = testProfile.Copy()
testProfiles[1] = testProfile.Copy()
wantFunc := []*profile.Function{
{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
}
wantLoc := []*profile.Location{
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
}
wantProfile := &profile.Profile{
SampleType: []*profile.ValueType{
&profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
},
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
Sample: []*profile.Sample{
{Location: []*profile.Location{wantLoc[0], wantLoc[1]}, Value: []int64{1}},
{Location: []*profile.Location{wantLoc[1]}, Value: []int64{1}},
},
Location: wantLoc,
Function: wantFunc,
}
// Profile already symbolized.
testProfiles[1].Location = []*profile.Location{
{ID: 1, Address: 0x10, Line: []profile.Line{{Function: wantFunc[0], Line: 10}}},
{ID: 2, Address: 0x20, Line: []profile.Line{{Function: wantFunc[1], Line: 20}}},
}
testProfiles[1].Function = []*profile.Function{
{ID: 1, Name: "foo", SystemName: "foo", Filename: "foo.go"},
{ID: 2, Name: "bar", SystemName: "bar", Filename: "bar.go"},
}
for i := 0; i < 2; i++ {
var prof bytes.Buffer
testProfiles[i].Write(&prof)
parseAndSymbolize(&prof)
gotProfile, err := profile.ParseData(prof.Bytes())
if err != nil {
t.Errorf("parsing symbolized profile (testcase = %d) got err: %v, want no error", i, err)
}
if !testutil.Equal(gotProfile, wantProfile, cmpOpt) {
t.Errorf("incorrect symbolization (testcase = %d): got {%v}, want {%v}", i, gotProfile, wantProfile)
}
}
}
func TestIsSymbolizedGoVersion(t *testing.T) {
for _, tc := range []struct {
input string
want bool
}{
{"go1.9beta2", true},
{"go1.9", true},
{"go1.9.1", true},
{"go1.10", true},
{"go1.10.1", true},
{"go2.0", true},
{"go3.1", true},
{"go1.8", false},
{"go1.8.1", false},
{"go1.7", false},
{"devel ", false},
} {
if got := isSymbolizedGoVersion(tc.input); got != tc.want {
t.Errorf("isSymbolizedGoVersion(%v) got %v, want %v", tc.input, got, tc.want)
}
}
}