initial commit

Signed-off-by: Derek McQuay <derekmcquay@gmail.com>
This commit is contained in:
dm
2018-04-10 19:17:26 -07:00
commit 59af8fc84f
2025 changed files with 456739 additions and 0 deletions
@@ -0,0 +1,66 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"fmt"
"time"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
func ExampleClient_Entries() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Entries(ctx, logadmin.Filter(`logName = "projects/my-project/logs/my-log"`))
_ = it // TODO: iterate using Next or iterator.Pager.
}
func ExampleFilter_timestamp() {
// This example demonstrates how to list the last 24 hours of log entries.
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
oneDayAgo := time.Now().Add(-24 * time.Hour)
t := oneDayAgo.Format(time.RFC3339) // Logging API wants timestamps in RFC 3339 format.
it := client.Entries(ctx, logadmin.Filter(fmt.Sprintf(`timestamp > "%s"`, t)))
_ = it // TODO: iterate using Next or iterator.Pager.
}
func ExampleEntryIterator_Next() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Entries(ctx)
for {
entry, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(entry)
}
}
@@ -0,0 +1,52 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"fmt"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
func ExampleClient_Metrics() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Metrics(ctx)
_ = it // TODO: iterate using Next or iterator.Pager.
}
func ExampleMetricIterator_Next() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Metrics(ctx)
for {
metric, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(metric)
}
}
+92
View File
@@ -0,0 +1,92 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"bytes"
"flag"
"fmt"
"html/template"
"log"
"net/http"
"cloud.google.com/go/logging"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
var (
client *logadmin.Client
projectID = flag.String("project-id", "", "ID of the project to use")
)
func ExampleClient_Entries_pagination() {
// This example demonstrates how to iterate through items a page at a time
// even if each successive page is fetched by a different process. It is a
// complete web server that displays pages of log entries. To run it as a
// standalone program, rename both the package and this function to "main".
ctx := context.Background()
flag.Parse()
if *projectID == "" {
log.Fatal("-project-id missing")
}
var err error
client, err = logadmin.NewClient(ctx, *projectID)
if err != nil {
log.Fatalf("creating logging client: %v", err)
}
http.HandleFunc("/entries", handleEntries)
log.Print("listening on 8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}
var pageTemplate = template.Must(template.New("").Parse(`
<table>
{{range .Entries}}
<tr><td>{{.}}</td></tr>
{{end}}
</table>
{{if .Next}}
<a href="/entries?pageToken={{.Next}}">Next Page</a>
{{end}}
`))
func handleEntries(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
filter := fmt.Sprintf(`logName = "projects/%s/logs/testlog"`, *projectID)
it := client.Entries(ctx, logadmin.Filter(filter))
var entries []*logging.Entry
nextTok, err := iterator.NewPager(it, 5, r.URL.Query().Get("pageToken")).NextPage(&entries)
if err != nil {
http.Error(w, fmt.Sprintf("problem getting the next page: %v", err), http.StatusInternalServerError)
return
}
data := struct {
Entries []*logging.Entry
Next string
}{
entries,
nextTok,
}
var buf bytes.Buffer
if err := pageTemplate.Execute(&buf, data); err != nil {
http.Error(w, fmt.Sprintf("problem executing page template: %v", err), http.StatusInternalServerError)
}
if _, err := buf.WriteTo(w); err != nil {
log.Printf("writing response: %v", err)
}
}
@@ -0,0 +1,52 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"fmt"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
func ExampleClient_ResourceDescriptors() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.ResourceDescriptors(ctx)
_ = it // TODO: iterate using Next or iterator.Pager.
}
func ExampleResourceDescriptorIterator_Next() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.ResourceDescriptors(ctx)
for {
rdesc, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(rdesc)
}
}
@@ -0,0 +1,52 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"fmt"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
func ExampleClient_Sinks() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Sinks(ctx)
_ = it // TODO: iterate using Next or iterator.Pager.
}
func ExampleSinkIterator_Next() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
it := client.Sinks(ctx)
for {
sink, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(sink)
}
}
+161
View File
@@ -0,0 +1,161 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin_test
import (
"fmt"
"cloud.google.com/go/logging/logadmin"
"golang.org/x/net/context"
)
func ExampleNewClient() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
// Use client to manage logs, metrics and sinks.
// Close the client when finished.
if err := client.Close(); err != nil {
// TODO: Handle error.
}
}
func ExampleClient_DeleteLog() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
err = client.DeleteLog(ctx, "my-log")
if err != nil {
// TODO: Handle error.
}
}
func ExampleClient_CreateMetric() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
err = client.CreateMetric(ctx, &logadmin.Metric{
ID: "severe-errors",
Description: "entries at ERROR or higher severities",
Filter: "severity >= ERROR",
})
if err != nil {
// TODO: Handle error.
}
}
func ExampleClient_DeleteMetric() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
if err := client.DeleteMetric(ctx, "severe-errors"); err != nil {
// TODO: Handle error.
}
}
func ExampleClient_Metric() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
m, err := client.Metric(ctx, "severe-errors")
if err != nil {
// TODO: Handle error.
}
fmt.Println(m)
}
func ExampleClient_UpdateMetric() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
err = client.UpdateMetric(ctx, &logadmin.Metric{
ID: "severe-errors",
Description: "entries at high severities",
Filter: "severity > ERROR",
})
if err != nil {
// TODO: Handle error.
}
}
func ExampleClient_CreateSink() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
sink, err := client.CreateSink(ctx, &logadmin.Sink{
ID: "severe-errors-to-gcs",
Destination: "storage.googleapis.com/my-bucket",
Filter: "severity >= ERROR",
})
if err != nil {
// TODO: Handle error.
}
fmt.Println(sink)
}
func ExampleClient_DeleteSink() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
if err := client.DeleteSink(ctx, "severe-errors-to-gcs"); err != nil {
// TODO: Handle error.
}
}
func ExampleClient_Sink() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
s, err := client.Sink(ctx, "severe-errors-to-gcs")
if err != nil {
// TODO: Handle error.
}
fmt.Println(s)
}
func ExampleClient_UpdateSink() {
ctx := context.Background()
client, err := logadmin.NewClient(ctx, "my-project")
if err != nil {
// TODO: Handle error.
}
sink, err := client.UpdateSink(ctx, &logadmin.Sink{
ID: "severe-errors-to-gcs",
Destination: "storage.googleapis.com/my-other-bucket",
Filter: "severity >= ERROR",
})
if err != nil {
// TODO: Handle error.
}
fmt.Println(sink)
}
+406
View File
@@ -0,0 +1,406 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// These features are missing now, but will likely be added:
// - There is no way to specify CallOptions.
// Package logadmin contains a Stackdriver Logging client that can be used
// for reading logs and working with sinks, metrics and monitored resources.
// For a client that can write logs, see package cloud.google.com/go/logging.
//
// The client uses Logging API v2.
// See https://cloud.google.com/logging/docs/api/v2/ for an introduction to the API.
//
// Note: This package is in beta. Some backwards-incompatible changes may occur.
package logadmin // import "cloud.google.com/go/logging/logadmin"
import (
"fmt"
"math"
"net/http"
"net/url"
"strings"
"time"
"cloud.google.com/go/internal/version"
"cloud.google.com/go/logging"
vkit "cloud.google.com/go/logging/apiv2"
"cloud.google.com/go/logging/internal"
"github.com/golang/protobuf/ptypes"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
logtypepb "google.golang.org/genproto/googleapis/logging/type"
logpb "google.golang.org/genproto/googleapis/logging/v2"
"google.golang.org/grpc/codes"
// Import the following so EntryIterator can unmarshal log protos.
_ "google.golang.org/genproto/googleapis/appengine/logging/v1"
_ "google.golang.org/genproto/googleapis/cloud/audit"
)
// Client is a Logging client. A Client is associated with a single Cloud project.
type Client struct {
lClient *vkit.Client // logging client
sClient *vkit.ConfigClient // sink client
mClient *vkit.MetricsClient // metric client
parent string
closed bool
}
// NewClient returns a new logging client associated with the provided project ID.
//
// By default NewClient uses AdminScope. To use a different scope, call
// NewClient using a WithScopes option (see https://godoc.org/google.golang.org/api/option#WithScopes).
func NewClient(ctx context.Context, parent string, opts ...option.ClientOption) (*Client, error) {
if !strings.ContainsRune(parent, '/') {
parent = "projects/" + parent
}
opts = append([]option.ClientOption{
option.WithEndpoint(internal.ProdAddr),
option.WithScopes(logging.AdminScope),
}, opts...)
lc, err := vkit.NewClient(ctx, opts...)
if err != nil {
return nil, err
}
// TODO(jba): pass along any client options that should be provided to all clients.
sc, err := vkit.NewConfigClient(ctx, option.WithGRPCConn(lc.Connection()))
if err != nil {
return nil, err
}
mc, err := vkit.NewMetricsClient(ctx, option.WithGRPCConn(lc.Connection()))
if err != nil {
return nil, err
}
// Retry some non-idempotent methods on INTERNAL, because it happens sometimes
// and in all observed cases the operation did not complete.
retryerOnInternal := func() gax.Retryer {
return gax.OnCodes([]codes.Code{
codes.Internal,
}, gax.Backoff{
Initial: 100 * time.Millisecond,
Max: 1000 * time.Millisecond,
Multiplier: 1.2,
})
}
mc.CallOptions.CreateLogMetric = []gax.CallOption{gax.WithRetry(retryerOnInternal)}
mc.CallOptions.UpdateLogMetric = []gax.CallOption{gax.WithRetry(retryerOnInternal)}
lc.SetGoogleClientInfo("gccl", version.Repo)
sc.SetGoogleClientInfo("gccl", version.Repo)
mc.SetGoogleClientInfo("gccl", version.Repo)
client := &Client{
lClient: lc,
sClient: sc,
mClient: mc,
parent: parent,
}
return client, nil
}
// Close closes the client.
func (c *Client) Close() error {
if c.closed {
return nil
}
// Return only the first error. Since all clients share an underlying connection,
// Closes after the first always report a "connection is closing" error.
err := c.lClient.Close()
_ = c.sClient.Close()
_ = c.mClient.Close()
c.closed = true
return err
}
// DeleteLog deletes a log and all its log entries. The log will reappear if it receives new entries.
// logID identifies the log within the project. An example log ID is "syslog". Requires AdminScope.
func (c *Client) DeleteLog(ctx context.Context, logID string) error {
return c.lClient.DeleteLog(ctx, &logpb.DeleteLogRequest{
LogName: internal.LogPath(c.parent, logID),
})
}
func toHTTPRequest(p *logtypepb.HttpRequest) (*logging.HTTPRequest, error) {
if p == nil {
return nil, nil
}
u, err := url.Parse(p.RequestUrl)
if err != nil {
return nil, err
}
var dur time.Duration
if p.Latency != nil {
dur, err = ptypes.Duration(p.Latency)
if err != nil {
return nil, err
}
}
hr := &http.Request{
Method: p.RequestMethod,
URL: u,
Header: map[string][]string{},
}
if p.UserAgent != "" {
hr.Header.Set("User-Agent", p.UserAgent)
}
if p.Referer != "" {
hr.Header.Set("Referer", p.Referer)
}
return &logging.HTTPRequest{
Request: hr,
RequestSize: p.RequestSize,
Status: int(p.Status),
ResponseSize: p.ResponseSize,
Latency: dur,
RemoteIP: p.RemoteIp,
CacheHit: p.CacheHit,
CacheValidatedWithOriginServer: p.CacheValidatedWithOriginServer,
}, nil
}
// An EntriesOption is an option for listing log entries.
type EntriesOption interface {
set(*logpb.ListLogEntriesRequest)
}
// ProjectIDs sets the project IDs or project numbers from which to retrieve
// log entries. Examples of a project ID: "my-project-1A", "1234567890".
func ProjectIDs(pids []string) EntriesOption { return projectIDs(pids) }
type projectIDs []string
func (p projectIDs) set(r *logpb.ListLogEntriesRequest) {
r.ResourceNames = make([]string, len(p))
for i, v := range p {
r.ResourceNames[i] = fmt.Sprintf("projects/%s", v)
}
}
// ResourceNames sets the resource names from which to retrieve
// log entries. Examples: "projects/my-project-1A", "organizations/my-org".
func ResourceNames(rns []string) EntriesOption { return resourceNames(rns) }
type resourceNames []string
func (rn resourceNames) set(r *logpb.ListLogEntriesRequest) {
r.ResourceNames = append([]string(nil), rn...)
}
// Filter sets an advanced logs filter for listing log entries (see
// https://cloud.google.com/logging/docs/view/advanced_filters). The filter is
// compared against all log entries in the projects specified by ProjectIDs.
// Only entries that match the filter are retrieved. An empty filter (the
// default) matches all log entries.
//
// In the filter string, log names must be written in their full form, as
// "projects/PROJECT-ID/logs/LOG-ID". Forward slashes in LOG-ID must be
// replaced by %2F before calling Filter.
//
// Timestamps in the filter string must be written in RFC 3339 format. See the
// timestamp example.
func Filter(f string) EntriesOption { return filter(f) }
type filter string
func (f filter) set(r *logpb.ListLogEntriesRequest) { r.Filter = string(f) }
// NewestFirst causes log entries to be listed from most recent (newest) to
// least recent (oldest). By default, they are listed from oldest to newest.
func NewestFirst() EntriesOption { return newestFirst{} }
type newestFirst struct{}
func (newestFirst) set(r *logpb.ListLogEntriesRequest) { r.OrderBy = "timestamp desc" }
// Entries returns an EntryIterator for iterating over log entries. By default,
// the log entries will be restricted to those from the project passed to
// NewClient. This may be overridden by passing a ProjectIDs option. Requires ReadScope or AdminScope.
func (c *Client) Entries(ctx context.Context, opts ...EntriesOption) *EntryIterator {
it := &EntryIterator{
it: c.lClient.ListLogEntries(ctx, listLogEntriesRequest(c.parent, opts)),
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
func listLogEntriesRequest(parent string, opts []EntriesOption) *logpb.ListLogEntriesRequest {
req := &logpb.ListLogEntriesRequest{
ResourceNames: []string{parent},
}
for _, opt := range opts {
opt.set(req)
}
return req
}
// An EntryIterator iterates over log entries.
type EntryIterator struct {
it *vkit.LogEntryIterator
pageInfo *iterator.PageInfo
nextFunc func() error
items []*logging.Entry
}
// PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
func (it *EntryIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is iterator.Done
// (https://godoc.org/google.golang.org/api/iterator) if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *EntryIterator) Next() (*logging.Entry, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *EntryIterator) fetch(pageSize int, pageToken string) (string, error) {
return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
item, err := it.it.Next()
if err != nil {
return err
}
e, err := fromLogEntry(item)
if err != nil {
return err
}
it.items = append(it.items, e)
return nil
})
}
func trunc32(i int) int32 {
if i > math.MaxInt32 {
i = math.MaxInt32
}
return int32(i)
}
var slashUnescaper = strings.NewReplacer("%2F", "/", "%2f", "/")
func fromLogEntry(le *logpb.LogEntry) (*logging.Entry, error) {
time, err := ptypes.Timestamp(le.Timestamp)
if err != nil {
return nil, err
}
var payload interface{}
switch x := le.Payload.(type) {
case *logpb.LogEntry_TextPayload:
payload = x.TextPayload
case *logpb.LogEntry_ProtoPayload:
var d ptypes.DynamicAny
if err := ptypes.UnmarshalAny(x.ProtoPayload, &d); err != nil {
return nil, fmt.Errorf("logging: unmarshalling proto payload: %v", err)
}
payload = d.Message
case *logpb.LogEntry_JsonPayload:
// Leave this as a Struct.
// TODO(jba): convert to map[string]interface{}?
payload = x.JsonPayload
default:
return nil, fmt.Errorf("logging: unknown payload type: %T", le.Payload)
}
hr, err := toHTTPRequest(le.HttpRequest)
if err != nil {
return nil, err
}
return &logging.Entry{
Timestamp: time,
Severity: logging.Severity(le.Severity),
Payload: payload,
Labels: le.Labels,
InsertID: le.InsertId,
HTTPRequest: hr,
Operation: le.Operation,
LogName: slashUnescaper.Replace(le.LogName),
Resource: le.Resource,
Trace: le.Trace,
}, nil
}
// Logs lists the logs owned by the parent resource of the client.
func (c *Client) Logs(ctx context.Context) *LogIterator {
it := &LogIterator{
parentResource: c.parent,
it: c.lClient.ListLogs(ctx, &logpb.ListLogsRequest{Parent: c.parent}),
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
// A LogIterator iterates over logs.
type LogIterator struct {
parentResource string
it *vkit.StringIterator
pageInfo *iterator.PageInfo
nextFunc func() error
items []string
}
// PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
func (it *LogIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is iterator.Done
// (https://godoc.org/google.golang.org/api/iterator) if there are no more
// results. Once Next returns Done, all subsequent calls will return Done.
func (it *LogIterator) Next() (string, error) {
if err := it.nextFunc(); err != nil {
return "", err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *LogIterator) fetch(pageSize int, pageToken string) (string, error) {
return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
logPath, err := it.it.Next()
if err != nil {
return err
}
logID := internal.LogIDFromPath(it.parentResource, logPath)
it.items = append(it.items, logID)
return nil
})
}
// Common fetch code for iterators that are backed by vkit iterators.
func iterFetch(pageSize int, pageToken string, pi *iterator.PageInfo, next func() error) (string, error) {
pi.MaxSize = pageSize
pi.Token = pageToken
// Get one item, which will fill the buffer.
if err := next(); err != nil {
return "", err
}
// Collect the rest of the buffer.
for pi.Remaining() > 0 {
if err := next(); err != nil {
return "", err
}
}
return pi.Token, nil
}
+267
View File
@@ -0,0 +1,267 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO(jba): test that OnError is getting called appropriately.
package logadmin
import (
"flag"
"log"
"net/http"
"net/url"
"os"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/logging"
ltesting "cloud.google.com/go/logging/internal/testing"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
durpb "github.com/golang/protobuf/ptypes/duration"
structpb "github.com/golang/protobuf/ptypes/struct"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/net/context"
"google.golang.org/api/option"
mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
audit "google.golang.org/genproto/googleapis/cloud/audit"
logtypepb "google.golang.org/genproto/googleapis/logging/type"
logpb "google.golang.org/genproto/googleapis/logging/v2"
"google.golang.org/grpc"
)
var (
client *Client
testProjectID string
)
var (
// If true, this test is using the production service, not a fake.
integrationTest bool
newClient func(ctx context.Context, projectID string) *Client
)
func TestMain(m *testing.M) {
flag.Parse() // needed for testing.Short()
ctx := context.Background()
testProjectID = testutil.ProjID()
if testProjectID == "" || testing.Short() {
integrationTest = false
if testProjectID != "" {
log.Print("Integration tests skipped in short mode (using fake instead)")
}
testProjectID = "PROJECT_ID"
addr, err := ltesting.NewServer()
if err != nil {
log.Fatalf("creating fake server: %v", err)
}
newClient = func(ctx context.Context, projectID string) *Client {
conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock())
if err != nil {
log.Fatalf("dialing %q: %v", addr, err)
}
c, err := NewClient(ctx, projectID, option.WithGRPCConn(conn))
if err != nil {
log.Fatalf("creating client for fake at %q: %v", addr, err)
}
return c
}
} else {
integrationTest = true
ts := testutil.TokenSource(ctx, logging.AdminScope)
if ts == nil {
log.Fatal("The project key must be set. See CONTRIBUTING.md for details")
}
log.Printf("running integration tests with project %s", testProjectID)
newClient = func(ctx context.Context, projectID string) *Client {
c, err := NewClient(ctx, projectID, option.WithTokenSource(ts),
option.WithGRPCDialOption(grpc.WithBlock()))
if err != nil {
log.Fatalf("creating prod client: %v", err)
}
return c
}
}
client = newClient(ctx, testProjectID)
initMetrics(ctx)
cleanup := initSinks(ctx)
exit := m.Run()
cleanup()
client.Close()
os.Exit(exit)
}
// EntryIterator and DeleteLog are tested in the logging package.
func TestClientClose(t *testing.T) {
c := newClient(context.Background(), testProjectID)
if err := c.Close(); err != nil {
t.Errorf("want got %v, want nil", err)
}
}
func TestFromLogEntry(t *testing.T) {
now := time.Now()
res := &mrpb.MonitoredResource{Type: "global"}
ts, err := ptypes.TimestampProto(now)
if err != nil {
t.Fatal(err)
}
logEntry := logpb.LogEntry{
LogName: "projects/PROJECT_ID/logs/LOG_ID",
Resource: res,
Payload: &logpb.LogEntry_TextPayload{TextPayload: "hello"},
Timestamp: ts,
Severity: logtypepb.LogSeverity_INFO,
InsertId: "123",
HttpRequest: &logtypepb.HttpRequest{
RequestMethod: "GET",
RequestUrl: "http:://example.com/path?q=1",
RequestSize: 100,
Status: 200,
ResponseSize: 25,
Latency: &durpb.Duration{Seconds: 100},
UserAgent: "user-agent",
RemoteIp: "127.0.0.1",
Referer: "referer",
CacheHit: true,
CacheValidatedWithOriginServer: true,
},
Labels: map[string]string{
"a": "1",
"b": "two",
"c": "true",
},
}
u, err := url.Parse("http:://example.com/path?q=1")
if err != nil {
t.Fatal(err)
}
want := &logging.Entry{
LogName: "projects/PROJECT_ID/logs/LOG_ID",
Resource: res,
Timestamp: now.In(time.UTC),
Severity: logging.Info,
Payload: "hello",
Labels: map[string]string{
"a": "1",
"b": "two",
"c": "true",
},
InsertID: "123",
HTTPRequest: &logging.HTTPRequest{
Request: &http.Request{
Method: "GET",
URL: u,
Header: map[string][]string{
"User-Agent": []string{"user-agent"},
"Referer": []string{"referer"},
},
},
RequestSize: 100,
Status: 200,
ResponseSize: 25,
Latency: 100 * time.Second,
RemoteIP: "127.0.0.1",
CacheHit: true,
CacheValidatedWithOriginServer: true,
},
}
got, err := fromLogEntry(&logEntry)
if err != nil {
t.Fatal(err)
}
if diff := testutil.Diff(got, want, cmpopts.IgnoreUnexported(http.Request{})); diff != "" {
t.Errorf("FullEntry:\n%s", diff)
}
// Proto payload.
alog := &audit.AuditLog{
ServiceName: "svc",
MethodName: "method",
ResourceName: "shelves/S/books/B",
}
any, err := ptypes.MarshalAny(alog)
if err != nil {
t.Fatal(err)
}
logEntry = logpb.LogEntry{
LogName: "projects/PROJECT_ID/logs/LOG_ID",
Resource: res,
Timestamp: ts,
Payload: &logpb.LogEntry_ProtoPayload{ProtoPayload: any},
}
got, err = fromLogEntry(&logEntry)
if err != nil {
t.Fatal(err)
}
if !ltesting.PayloadEqual(got.Payload, alog) {
t.Errorf("got %+v, want %+v", got.Payload, alog)
}
// JSON payload.
jstruct := &structpb.Struct{Fields: map[string]*structpb.Value{
"f": &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: 3.1}},
}}
logEntry = logpb.LogEntry{
LogName: "projects/PROJECT_ID/logs/LOG_ID",
Resource: res,
Timestamp: ts,
Payload: &logpb.LogEntry_JsonPayload{JsonPayload: jstruct},
}
got, err = fromLogEntry(&logEntry)
if err != nil {
t.Fatal(err)
}
if !ltesting.PayloadEqual(got.Payload, jstruct) {
t.Errorf("got %+v, want %+v", got.Payload, jstruct)
}
}
func TestListLogEntriesRequest(t *testing.T) {
for _, test := range []struct {
opts []EntriesOption
resourceNames []string
filter string
orderBy string
}{
// Default is client's project ID, empty filter and orderBy.
{nil, []string{"projects/PROJECT_ID"}, "", ""},
{[]EntriesOption{NewestFirst(), Filter("f")},
[]string{"projects/PROJECT_ID"}, "f", "timestamp desc"},
{[]EntriesOption{ProjectIDs([]string{"foo"})},
[]string{"projects/foo"}, "", ""},
{[]EntriesOption{ResourceNames([]string{"folders/F", "organizations/O"})},
[]string{"folders/F", "organizations/O"}, "", ""},
{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
[]string{"projects/foo"}, "f", "timestamp desc"},
{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
[]string{"projects/foo"}, "f", "timestamp desc"},
// If there are repeats, last one wins.
{[]EntriesOption{NewestFirst(), Filter("no"), ProjectIDs([]string{"foo"}), Filter("f")},
[]string{"projects/foo"}, "f", "timestamp desc"},
} {
got := listLogEntriesRequest("projects/PROJECT_ID", test.opts)
want := &logpb.ListLogEntriesRequest{
ResourceNames: test.resourceNames,
Filter: test.filter,
OrderBy: test.orderBy,
}
if !proto.Equal(got, want) {
t.Errorf("%v:\ngot %v\nwant %v", test.opts, got, want)
}
}
}
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin
import (
"fmt"
vkit "cloud.google.com/go/logging/apiv2"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
logpb "google.golang.org/genproto/googleapis/logging/v2"
)
// Metric describes a logs-based metric. The value of the metric is the
// number of log entries that match a logs filter.
//
// Metrics are a feature of Stackdriver Monitoring.
// See https://cloud.google.com/monitoring/api/v3/metrics for more about them.
type Metric struct {
// ID is a client-assigned metric identifier. Example:
// "severe_errors". Metric identifiers are limited to 1000
// characters and can include only the following characters: A-Z,
// a-z, 0-9, and the special characters _-.,+!*',()%/\. The
// forward-slash character (/) denotes a hierarchy of name pieces,
// and it cannot be the first character of the name.
ID string
// Description describes this metric. It is used in documentation.
Description string
// Filter is an advanced logs filter (see
// https://cloud.google.com/logging/docs/view/advanced_filters).
// Example: "logName:syslog AND severity>=ERROR".
Filter string
}
// CreateMetric creates a logs-based metric.
func (c *Client) CreateMetric(ctx context.Context, m *Metric) error {
_, err := c.mClient.CreateLogMetric(ctx, &logpb.CreateLogMetricRequest{
Parent: c.parent,
Metric: toLogMetric(m),
})
return err
}
// DeleteMetric deletes a log-based metric.
// The provided metric ID is the metric identifier. For example, "severe_errors".
func (c *Client) DeleteMetric(ctx context.Context, metricID string) error {
return c.mClient.DeleteLogMetric(ctx, &logpb.DeleteLogMetricRequest{
MetricName: c.metricPath(metricID),
})
}
// Metric gets a logs-based metric.
// The provided metric ID is the metric identifier. For example, "severe_errors".
// Requires ReadScope or AdminScope.
func (c *Client) Metric(ctx context.Context, metricID string) (*Metric, error) {
lm, err := c.mClient.GetLogMetric(ctx, &logpb.GetLogMetricRequest{
MetricName: c.metricPath(metricID),
})
if err != nil {
return nil, err
}
return fromLogMetric(lm), nil
}
// UpdateMetric creates a logs-based metric if it does not exist, or updates an
// existing one.
func (c *Client) UpdateMetric(ctx context.Context, m *Metric) error {
_, err := c.mClient.UpdateLogMetric(ctx, &logpb.UpdateLogMetricRequest{
MetricName: c.metricPath(m.ID),
Metric: toLogMetric(m),
})
return err
}
func (c *Client) metricPath(metricID string) string {
return fmt.Sprintf("%s/metrics/%s", c.parent, metricID)
}
// Metrics returns a MetricIterator for iterating over all Metrics in the Client's project.
// Requires ReadScope or AdminScope.
func (c *Client) Metrics(ctx context.Context) *MetricIterator {
it := &MetricIterator{
it: c.mClient.ListLogMetrics(ctx, &logpb.ListLogMetricsRequest{Parent: c.parent}),
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
// A MetricIterator iterates over Metrics.
type MetricIterator struct {
it *vkit.LogMetricIterator
pageInfo *iterator.PageInfo
nextFunc func() error
items []*Metric
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *MetricIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is Done if there are
// no more results. Once Next returns Done, all subsequent calls will return
// Done.
func (it *MetricIterator) Next() (*Metric, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *MetricIterator) fetch(pageSize int, pageToken string) (string, error) {
return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
item, err := it.it.Next()
if err != nil {
return err
}
it.items = append(it.items, fromLogMetric(item))
return nil
})
}
func toLogMetric(m *Metric) *logpb.LogMetric {
return &logpb.LogMetric{
Name: m.ID,
Description: m.Description,
Filter: m.Filter,
}
}
func fromLogMetric(lm *logpb.LogMetric) *Metric {
return &Metric{
ID: lm.Name,
Description: lm.Description,
Filter: lm.Filter,
}
}
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin
import (
"log"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
var metricIDs = testutil.NewUIDSpace("GO-CLIENT-TEST-METRIC")
// Initializes the tests before they run.
func initMetrics(ctx context.Context) {
// Clean up from aborted tests.
it := client.Metrics(ctx)
loop:
for {
m, err := it.Next()
switch err {
case nil:
if metricIDs.Older(m.ID, 24*time.Hour) {
client.DeleteMetric(ctx, m.ID)
}
case iterator.Done:
break loop
default:
log.Printf("cleanupMetrics: %v", err)
return
}
}
}
func TestCreateDeleteMetric(t *testing.T) {
ctx := context.Background()
metric := &Metric{
ID: metricIDs.New(),
Description: "DESC",
Filter: "FILTER",
}
if err := client.CreateMetric(ctx, metric); err != nil {
t.Fatal(err)
}
defer client.DeleteMetric(ctx, metric.ID)
got, err := client.Metric(ctx, metric.ID)
if err != nil {
t.Fatal(err)
}
if want := metric; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
if err := client.DeleteMetric(ctx, metric.ID); err != nil {
t.Fatal(err)
}
if _, err := client.Metric(ctx, metric.ID); err == nil {
t.Fatal("got no error, expected one")
}
}
func TestUpdateMetric(t *testing.T) {
ctx := context.Background()
metric := &Metric{
ID: metricIDs.New(),
Description: "DESC",
Filter: "FILTER",
}
// Updating a non-existent metric creates a new one.
if err := client.UpdateMetric(ctx, metric); err != nil {
t.Fatal(err)
}
defer client.DeleteMetric(ctx, metric.ID)
got, err := client.Metric(ctx, metric.ID)
if err != nil {
t.Fatal(err)
}
if want := metric; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
// Updating an existing metric changes it.
metric.Description = "CHANGED"
if err := client.UpdateMetric(ctx, metric); err != nil {
t.Fatal(err)
}
got, err = client.Metric(ctx, metric.ID)
if err != nil {
t.Fatal(err)
}
if want := metric; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}
func TestListMetrics(t *testing.T) {
ctx := context.Background()
var metrics []*Metric
want := map[string]*Metric{}
for i := 0; i < 10; i++ {
m := &Metric{
ID: metricIDs.New(),
Description: "DESC",
Filter: "FILTER",
}
metrics = append(metrics, m)
want[m.ID] = m
}
for _, m := range metrics {
if err := client.CreateMetric(ctx, m); err != nil {
t.Fatalf("Create(%q): %v", m.ID, err)
}
defer client.DeleteMetric(ctx, m.ID)
}
got := map[string]*Metric{}
it := client.Metrics(ctx)
for {
m, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
t.Fatal(err)
}
// If tests run simultaneously, we may have more metrics than we
// created. So only check for our own.
if _, ok := want[m.ID]; ok {
got[m.ID] = m
}
}
if !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin
import (
vkit "cloud.google.com/go/logging/apiv2"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
logpb "google.golang.org/genproto/googleapis/logging/v2"
)
// ResourceDescriptors returns a ResourceDescriptorIterator
// for iterating over MonitoredResourceDescriptors. Requires ReadScope or AdminScope.
// See https://cloud.google.com/logging/docs/api/v2/#monitored-resources for an explanation of
// monitored resources.
// See https://cloud.google.com/logging/docs/api/v2/resource-list for a list of monitored resources.
func (c *Client) ResourceDescriptors(ctx context.Context) *ResourceDescriptorIterator {
it := &ResourceDescriptorIterator{
it: c.lClient.ListMonitoredResourceDescriptors(ctx,
&logpb.ListMonitoredResourceDescriptorsRequest{}),
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
// ResourceDescriptorIterator is an iterator over MonitoredResourceDescriptors.
type ResourceDescriptorIterator struct {
it *vkit.MonitoredResourceDescriptorIterator
pageInfo *iterator.PageInfo
nextFunc func() error
items []*mrpb.MonitoredResourceDescriptor
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *ResourceDescriptorIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is Done if there are
// no more results. Once Next returns Done, all subsequent calls will return
// Done.
func (it *ResourceDescriptorIterator) Next() (*mrpb.MonitoredResourceDescriptor, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *ResourceDescriptorIterator) fetch(pageSize int, pageToken string) (string, error) {
return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
item, err := it.it.Next()
if err != nil {
return err
}
it.items = append(it.items, item)
return nil
})
}
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin
import (
"testing"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
func TestMonitoredResourceDescriptors(t *testing.T) {
// We can't create MonitoredResourceDescriptors, and there is no guarantee
// about what the service will return. So we just check that the result is
// non-empty.
it := client.ResourceDescriptors(context.Background())
n := 0
loop:
for {
_, err := it.Next()
switch err {
case nil:
n++
case iterator.Done:
break loop
default:
t.Fatal(err)
}
}
if n == 0 {
t.Fatal("Next: got no MetricResourceDescriptors, expected at least one")
}
// TODO(jba) test pagination.
}
+168
View File
@@ -0,0 +1,168 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package logadmin
import (
"fmt"
vkit "cloud.google.com/go/logging/apiv2"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
logpb "google.golang.org/genproto/googleapis/logging/v2"
)
// Sink describes a sink used to export log entries outside Stackdriver
// Logging. Incoming log entries matching a filter are exported to a
// destination (a Cloud Storage bucket, BigQuery dataset or Cloud Pub/Sub
// topic).
//
// For more information, see https://cloud.google.com/logging/docs/export/using_exported_logs.
// (The Sinks in this package are what the documentation refers to as "project sinks".)
type Sink struct {
// ID is a client-assigned sink identifier. Example:
// "my-severe-errors-to-pubsub".
// Sink identifiers are limited to 1000 characters
// and can include only the following characters: A-Z, a-z,
// 0-9, and the special characters "_-.".
ID string
// Destination is the export destination. See
// https://cloud.google.com/logging/docs/api/tasks/exporting-logs.
// Examples: "storage.googleapis.com/a-bucket",
// "bigquery.googleapis.com/projects/a-project-id/datasets/a-dataset".
Destination string
// Filter optionally specifies an advanced logs filter (see
// https://cloud.google.com/logging/docs/view/advanced_filters) that
// defines the log entries to be exported. Example: "logName:syslog AND
// severity>=ERROR". If omitted, all entries are returned.
Filter string
}
// CreateSink creates a Sink. It returns an error if the Sink already exists.
// Requires AdminScope.
func (c *Client) CreateSink(ctx context.Context, sink *Sink) (*Sink, error) {
ls, err := c.sClient.CreateSink(ctx, &logpb.CreateSinkRequest{
Parent: c.parent,
Sink: toLogSink(sink),
})
if err != nil {
fmt.Printf("Sink: %+v\n", toLogSink(sink))
return nil, err
}
return fromLogSink(ls), nil
}
// DeleteSink deletes a sink. The provided sinkID is the sink's identifier, such as
// "my-severe-errors-to-pubsub".
// Requires AdminScope.
func (c *Client) DeleteSink(ctx context.Context, sinkID string) error {
return c.sClient.DeleteSink(ctx, &logpb.DeleteSinkRequest{
SinkName: c.sinkPath(sinkID),
})
}
// Sink gets a sink. The provided sinkID is the sink's identifier, such as
// "my-severe-errors-to-pubsub".
// Requires ReadScope or AdminScope.
func (c *Client) Sink(ctx context.Context, sinkID string) (*Sink, error) {
ls, err := c.sClient.GetSink(ctx, &logpb.GetSinkRequest{
SinkName: c.sinkPath(sinkID),
})
if err != nil {
return nil, err
}
return fromLogSink(ls), nil
}
// UpdateSink updates an existing Sink. Requires AdminScope.
func (c *Client) UpdateSink(ctx context.Context, sink *Sink) (*Sink, error) {
ls, err := c.sClient.UpdateSink(ctx, &logpb.UpdateSinkRequest{
SinkName: c.sinkPath(sink.ID),
Sink: toLogSink(sink),
})
if err != nil {
return nil, err
}
return fromLogSink(ls), err
}
func (c *Client) sinkPath(sinkID string) string {
return fmt.Sprintf("%s/sinks/%s", c.parent, sinkID)
}
// Sinks returns a SinkIterator for iterating over all Sinks in the Client's project.
// Requires ReadScope or AdminScope.
func (c *Client) Sinks(ctx context.Context) *SinkIterator {
it := &SinkIterator{
it: c.sClient.ListSinks(ctx, &logpb.ListSinksRequest{Parent: c.parent}),
}
it.pageInfo, it.nextFunc = iterator.NewPageInfo(
it.fetch,
func() int { return len(it.items) },
func() interface{} { b := it.items; it.items = nil; return b })
return it
}
// A SinkIterator iterates over Sinks.
type SinkIterator struct {
it *vkit.LogSinkIterator
pageInfo *iterator.PageInfo
nextFunc func() error
items []*Sink
}
// PageInfo supports pagination. See the google.golang.org/api/iterator package for details.
func (it *SinkIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
// Next returns the next result. Its second return value is Done if there are
// no more results. Once Next returns Done, all subsequent calls will return
// Done.
func (it *SinkIterator) Next() (*Sink, error) {
if err := it.nextFunc(); err != nil {
return nil, err
}
item := it.items[0]
it.items = it.items[1:]
return item, nil
}
func (it *SinkIterator) fetch(pageSize int, pageToken string) (string, error) {
return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
item, err := it.it.Next()
if err != nil {
return err
}
it.items = append(it.items, fromLogSink(item))
return nil
})
}
func toLogSink(s *Sink) *logpb.LogSink {
return &logpb.LogSink{
Name: s.ID,
Destination: s.Destination,
Filter: s.Filter,
OutputVersionFormat: logpb.LogSink_V2,
}
}
func fromLogSink(ls *logpb.LogSink) *Sink {
return &Sink{
ID: ls.Name,
Destination: ls.Destination,
Filter: ls.Filter,
}
}
+227
View File
@@ -0,0 +1,227 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO(jba): document in CONTRIBUTING.md that service account must be given "Logs Configuration Writer" IAM role for sink tests to pass.
// TODO(jba): [cont] (1) From top left menu, go to IAM & Admin. (2) In Roles dropdown for acct, select Logging > Logs Configuration Writer. (3) Save.
// TODO(jba): Also, cloud-logs@google.com must have Owner permission on the GCS bucket named for the test project.
package logadmin
import (
"log"
"testing"
"time"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/storage"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
)
var sinkIDs = testutil.NewUIDSpace("GO-CLIENT-TEST-SINK")
const testFilter = ""
var testSinkDestination string
// Called just before TestMain calls m.Run.
// Returns a cleanup function to be called after the tests finish.
func initSinks(ctx context.Context) func() {
// Create a unique GCS bucket so concurrent tests don't interfere with each other.
bucketIDs := testutil.NewUIDSpace(testProjectID + "-log-sink")
testBucket := bucketIDs.New()
testSinkDestination = "storage.googleapis.com/" + testBucket
var storageClient *storage.Client
if integrationTest {
// Create a unique bucket as a sink destination, and give the cloud logging account
// owner right.
ts := testutil.TokenSource(ctx, storage.ScopeFullControl)
var err error
storageClient, err = storage.NewClient(ctx, option.WithTokenSource(ts))
if err != nil {
log.Fatalf("new storage client: %v", err)
}
bucket := storageClient.Bucket(testBucket)
if err := bucket.Create(ctx, testProjectID, nil); err != nil {
log.Fatalf("creating storage bucket %q: %v", testBucket, err)
}
if err := bucket.ACL().Set(ctx, "group-cloud-logs@google.com", storage.RoleOwner); err != nil {
log.Fatalf("setting owner role: %v", err)
}
}
// Clean up from aborted tests.
it := client.Sinks(ctx)
for {
s, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
log.Printf("listing sinks: %v", err)
break
}
if sinkIDs.Older(s.ID, 24*time.Hour) {
client.DeleteSink(ctx, s.ID) // ignore error
}
}
if integrationTest {
for _, bn := range bucketNames(ctx, storageClient) {
if bucketIDs.Older(bn, 24*time.Hour) {
storageClient.Bucket(bn).Delete(ctx) // ignore error
}
}
return func() {
if err := storageClient.Bucket(testBucket).Delete(ctx); err != nil {
log.Printf("deleting %q: %v", testBucket, err)
}
storageClient.Close()
}
}
return func() {}
}
// Collect the name of all buckets for the test project.
func bucketNames(ctx context.Context, client *storage.Client) []string {
var names []string
it := client.Buckets(ctx, testProjectID)
loop:
for {
b, err := it.Next()
switch err {
case nil:
names = append(names, b.Name)
case iterator.Done:
break loop
default:
log.Printf("listing buckets: %v", err)
break loop
}
}
return names
}
func TestCreateDeleteSink(t *testing.T) {
ctx := context.Background()
sink := &Sink{
ID: sinkIDs.New(),
Destination: testSinkDestination,
Filter: testFilter,
}
got, err := client.CreateSink(ctx, sink)
if err != nil {
t.Fatal(err)
}
defer client.DeleteSink(ctx, sink.ID)
if want := sink; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
got, err = client.Sink(ctx, sink.ID)
if err != nil {
t.Fatal(err)
}
if want := sink; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
if err := client.DeleteSink(ctx, sink.ID); err != nil {
t.Fatal(err)
}
if _, err := client.Sink(ctx, sink.ID); err == nil {
t.Fatal("got no error, expected one")
}
}
func TestUpdateSink(t *testing.T) {
ctx := context.Background()
sink := &Sink{
ID: sinkIDs.New(),
Destination: testSinkDestination,
Filter: testFilter,
}
if _, err := client.CreateSink(ctx, sink); err != nil {
t.Fatal(err)
}
got, err := client.UpdateSink(ctx, sink)
if err != nil {
t.Fatal(err)
}
defer client.DeleteSink(ctx, sink.ID)
if want := sink; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
got, err = client.Sink(ctx, sink.ID)
if err != nil {
t.Fatal(err)
}
if want := sink; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
// Updating an existing sink changes it.
sink.Filter = ""
if _, err := client.UpdateSink(ctx, sink); err != nil {
t.Fatal(err)
}
got, err = client.Sink(ctx, sink.ID)
if err != nil {
t.Fatal(err)
}
if want := sink; !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}
func TestListSinks(t *testing.T) {
ctx := context.Background()
var sinks []*Sink
want := map[string]*Sink{}
for i := 0; i < 4; i++ {
s := &Sink{
ID: sinkIDs.New(),
Destination: testSinkDestination,
Filter: testFilter,
}
sinks = append(sinks, s)
want[s.ID] = s
}
for _, s := range sinks {
if _, err := client.CreateSink(ctx, s); err != nil {
t.Fatalf("Create(%q): %v", s.ID, err)
}
defer client.DeleteSink(ctx, s.ID)
}
got := map[string]*Sink{}
it := client.Sinks(ctx)
for {
s, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
t.Fatal(err)
}
// If tests run simultaneously, we may have more sinks than we
// created. So only check for our own.
if _, ok := want[s.ID]; ok {
got[s.ID] = s
}
}
if !testutil.Equal(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}