census.h

Go to the documentation of this file.

/*
 *
 * Copyright 2015-2016, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/* RPC-internal Census API's. These are designed to be generic enough that
 * they can (ultimately) be used in many different RPC systems (with differing
 * implementations). */

#ifndef CENSUS_CENSUS_H
#define CENSUS_CENSUS_H

#include <grpc/grpc.h>

#ifdef __cplusplus
extern "C" {
#endif

/* Identify census features that can be enabled via census_initialize(). */
enum census_features {
  CENSUS_FEATURE_NONE = 0,    /* Do not enable census. */
  CENSUS_FEATURE_TRACING = 1, /* Enable census tracing. */
  CENSUS_FEATURE_STATS = 2,   /* Enable Census stats collection. */
  CENSUS_FEATURE_CPU = 4,     /* Enable Census CPU usage collection. */
  CENSUS_FEATURE_ALL =
      CENSUS_FEATURE_TRACING | CENSUS_FEATURE_STATS | CENSUS_FEATURE_CPU
};

CENSUSAPI int census_initialize(int features);
CENSUSAPI void census_shutdown(void);

CENSUSAPI int census_supported(void);

CENSUSAPI int census_enabled(void);

typedef struct census_context census_context;

/* A tag is a key:value pair. The key is a non-empty, printable (UTF-8
   encoded), nil-terminated string. The value is a binary string, that may be
   printable. There are limits on the sizes of both keys and values (see
   CENSUS_MAX_TAG_KB_LEN definition below), and the number of tags that can be
   propagated (CENSUS_MAX_PROPAGATED_TAGS). Users should also remember that
   some systems may have limits on, e.g., the number of bytes that can be
   transmitted as metadata, and that larger tags means more memory consumed
   and time in processing. */
typedef struct {
  const char *key;
  const char *value;
  size_t value_len;
  uint8_t flags;
} census_tag;

/* Maximum length of a tag's key or value. */
#define CENSUS_MAX_TAG_KV_LEN 255
/* Maximum number of propagatable tags. */
#define CENSUS_MAX_PROPAGATED_TAGS 255

/* Tag flags. */
#define CENSUS_TAG_PROPAGATE 1 /* Tag should be propagated over RPC */
#define CENSUS_TAG_STATS 2     /* Tag will be used for statistics aggregation */
#define CENSUS_TAG_BINARY 4    /* Tag value is not printable */
#define CENSUS_TAG_RESERVED 8  /* Reserved for internal use. */
/* Flag values 8,16,32,64,128 are reserved for future/internal use. Clients
   should not use or rely on their values. */

#define CENSUS_TAG_IS_PROPAGATED(flags) (flags & CENSUS_TAG_PROPAGATE)
#define CENSUS_TAG_IS_STATS(flags) (flags & CENSUS_TAG_STATS)
#define CENSUS_TAG_IS_BINARY(flags) (flags & CENSUS_TAG_BINARY)

/* An instance of this structure is kept by every context, and records the
   basic information associated with the creation of that context. */
typedef struct {
  int n_propagated_tags;        /* number of propagated printable tags */
  int n_propagated_binary_tags; /* number of propagated binary tags */
  int n_local_tags;             /* number of non-propagated (local) tags */
  int n_deleted_tags;           /* number of tags that were deleted */
  int n_added_tags;             /* number of tags that were added */
  int n_modified_tags;          /* number of tags that were modified */
  int n_invalid_tags;           /* number of tags with bad keys or values (e.g.
                                   longer than CENSUS_MAX_TAG_KV_LEN) */
  int n_ignored_tags;           /* number of tags ignored because of
                                   CENSUS_MAX_PROPAGATED_TAGS limit. */
} census_context_status;

/* Create a new context, adding and removing tags from an existing context.
   This will copy all tags from the 'tags' input, so it is recommended
   to add as many tags in a single operation as is practical for the client.
   @param base Base context to build upon. Can be NULL.
   @param tags A set of tags to be added/changed/deleted. Tags with keys that
   are in 'tags', but not 'base', are added to the tag set. Keys that are in
   both 'tags' and 'base' will have their value/flags modified. Tags with keys
   in both, but with NULL or zero-length values, will be deleted from the tag
   set. Tags with invalid (too long or short) keys or values will be ignored.
   If adding a tag will result in more than CENSUS_MAX_PROPAGATED_TAGS in either
   binary or non-binary tags, they will be ignored, as will deletions of
   tags that don't exist.
   @param ntags number of tags in 'tags'
   @param status If not NULL, will return a pointer to a census_context_status
   structure containing information about the new context and status of the
   tags used in its creation.
   @return A new, valid census_context.
*/
CENSUSAPI census_context *census_context_create(
    const census_context *base, const census_tag *tags, int ntags,
    census_context_status const **status);

/* Destroy a context. Once this function has been called, the context cannot
   be reused. */
CENSUSAPI void census_context_destroy(census_context *context);

/* Get a pointer to the original status from the context creation. */
CENSUSAPI const census_context_status *census_context_get_status(
    const census_context *context);

/* Structure used for iterating over the tegs in a context. API clients should
   not use or reference internal fields - neither their contents or
   presence/absence are guaranteed. */
typedef struct {
  const census_context *context;
  int base;
  int index;
  char *kvm;
} census_context_iterator;

/* Initialize a census_tag_iterator. Must be called before first use. */
CENSUSAPI void census_context_initialize_iterator(
    const census_context *context, census_context_iterator *iterator);

/* Get the contents of the "next" tag in the context. If there are no more
   tags, returns 0 (and 'tag' contents will be unchanged), otherwise returns 1.
   */
CENSUSAPI int census_context_next_tag(census_context_iterator *iterator,
                                      census_tag *tag);

/* Get a context tag by key. Returns 0 if the key is not present. */
CENSUSAPI int census_context_get_tag(const census_context *context,
                                     const char *key, census_tag *tag);

/* Tag set encode/decode functionality. These functionas are intended
   for use by RPC systems only, for purposes of transmitting/receiving contexts.
   */

/* Encode a context into a buffer. The propagated tags are encoded into the
   buffer in two regions: one for printable tags, and one for binary tags.
   @param context context to be encoded
   @param buffer pointer to buffer. This address will be used to encode the
                 printable tags.
   @param buf_size number of available bytes in buffer.
   @param print_buf_size Will be set to the number of bytes consumed by
                         printable tags.
   @param bin_buf_size Will be set to the number of bytes used to encode the
                       binary tags.
   @return A pointer to the binary tag's encoded, or NULL if the buffer was
           insufficiently large to hold the encoded tags. Thus, if successful,
           printable tags are encoded into
           [buffer, buffer + *print_buf_size) and binary tags into
           [returned-ptr, returned-ptr + *bin_buf_size) (and the returned
           pointer should be buffer + *print_buf_size) */
CENSUSAPI char *census_context_encode(const census_context *context,
                                      char *buffer, size_t buf_size,
                                      size_t *print_buf_size,
                                      size_t *bin_buf_size);

/* Decode context buffers encoded with census_context_encode(). Returns NULL
   if there is an error in parsing either buffer. */
CENSUSAPI census_context *census_context_decode(const char *buffer, size_t size,
                                                const char *bin_buffer,
                                                size_t bin_size);

/* Distributed traces can have a number of options. */
enum census_trace_mask_values {
  CENSUS_TRACE_MASK_NONE = 0,      /* Default, empty flags */
  CENSUS_TRACE_MASK_IS_SAMPLED = 1 /* RPC tracing enabled for this context. */
};

CENSUSAPI int census_trace_mask(const census_context *context);

CENSUSAPI void census_set_trace_mask(int trace_mask);

/* The concept of "operation" is a fundamental concept for Census. In an RPC
   system, and operation typcially represents a single RPC, or a significant
   sub-part thereof (e.g. a single logical "read" RPC to a distributed storage
   system might do several other actions in parallel, from looking up metadata
   indices to making requests of other services - each of these could be a
   sub-operation with the larger RPC operation). Census uses operations for the
   following:

   CPU accounting: If enabled, census will measure the thread CPU time
   consumed between operation start and end times.

   Active operations: Census will maintain information on all currently
   active operations.

   Distributed tracing: Each operation serves as a logical trace span.

   Stats collection: Stats are broken down by operation (e.g. latency
   breakdown for each unique RPC path).

   The following functions serve to delineate the start and stop points for
   each logical operation. */

typedef struct {
  /* Use gpr_timespec for default implementation. High performance
   * implementations should use a cycle-counter based timestamp. */
  gpr_timespec ts;
} census_timestamp;

CENSUSAPI census_timestamp census_start_rpc_op_timestamp(void);

typedef struct {
  const char *(*get_rpc_service_name)(int64_t id);
  const char *(*get_rpc_method_name)(int64_t id);
} census_rpc_name_info;

CENSUSAPI census_context *census_start_client_rpc_op(
    const census_context *context, int64_t rpc_name_id,
    const census_rpc_name_info *rpc_name_info, const char *peer, int trace_mask,
    const census_timestamp *start_time);

CENSUSAPI void census_set_rpc_client_peer(census_context *context,
                                          const char *peer);

CENSUSAPI census_context *census_start_server_rpc_op(
    const char *buffer, int64_t rpc_name_id,
    const census_rpc_name_info *rpc_name_info, const char *peer, int trace_mask,
    census_timestamp *start_time);

CENSUSAPI census_context *census_start_op(census_context *context,
                                          const char *family, const char *name,
                                          int trace_mask);

CENSUSAPI void census_end_op(census_context *context, int status);

#define CENSUS_TRACE_RECORD_START_OP ((uint32_t)0)
#define CENSUS_TRACE_RECORD_END_OP ((uint32_t)1)

CENSUSAPI void census_trace_print(census_context *context, uint32_t type,
                                  const char *buffer, size_t n);

typedef struct {
  census_timestamp timestamp; /* Time of record creation */
  uint64_t trace_id;          /* Trace ID associated with record */
  uint64_t op_id;             /* Operation ID associated with record */
  uint32_t type;              /* Type (as used in census_trace_print() */
  const char *buffer;         /* Buffer (from census_trace_print() */
  size_t buf_size;            /* Number of bytes inside buffer */
} census_trace_record;

CENSUSAPI int census_trace_scan_start(int consume);

CENSUSAPI int census_get_trace_record(census_trace_record *trace_record);

CENSUSAPI void census_trace_scan_end();

/* Core stats collection API's. The following concepts are used:
   * Aggregation: A collection of values. Census supports the following
       aggregation types:
         Sum - a single summation type. Typically used for keeping (e.g.)
           counts of events.
         Distribution - statistical distribution information, used for
           recording average, standard deviation etc.
         Histogram - a histogram of measurements falling in defined bucket
           boundaries.
         Window - a count of events that happen in reolling time window.
     New aggregation types can be added by the user, if desired (see
     census_register_aggregation()).
   * Metric: Each measurement is for a single metric. Examples include RPC
     latency, CPU seconds consumed, and bytes transmitted.
   * View: A view is a combination of a metric, a tag set (in which the tag
     values are regular expressions) and a set of aggregations. When a
     measurement for a metric matches the view tags, it is recorded (for each
     unique set of tags) against each aggregation. Each metric can have an
     arbitrary number of views by which it will be broken down.
*/

/* A single value to be recorded comprises two parts: an ID for the particular
 * metric and the value to be recorded against it. */
typedef struct {
  uint32_t metric_id;
  double value;
} census_value;

/* Record new usage values against the given context. */
CENSUSAPI void census_record_values(census_context *context,
                                    census_value *values, size_t nvalues);

typedef struct census_aggregation_ops census_aggregation_ops;

/* Predefined aggregation types, for use with census_view_create(). */
extern census_aggregation_ops census_agg_sum;
extern census_aggregation_ops census_agg_distribution;
extern census_aggregation_ops census_agg_histogram;
extern census_aggregation_ops census_agg_window;

typedef struct {
  const census_aggregation_ops *ops;
  const void *create_arg; /* Aaggregation initialization argument. */
} census_aggregation;

typedef struct census_view census_view;

/* TODO(aveitch): consider if context is the right argument type to pass in
   tags. */
CENSUSAPI census_view *census_view_create(
    uint32_t metric_id, const census_context *tags,
    const census_aggregation *aggregations, size_t naggregations);

CENSUSAPI void census_view_delete(census_view *view);

CENSUSAPI size_t census_view_metric(const census_view *view);

CENSUSAPI size_t census_view_naggregations(const census_view *view);

CENSUSAPI const census_context *census_view_tags(const census_view *view);

CENSUSAPI const census_aggregation *census_view_aggregrations(
    const census_view *view);

typedef struct {
  const census_context *tags; /* Tags for this set of aggregations. */
  const void **data; /* One data set for every aggregation in the view. */
} census_view_aggregation_data;

typedef struct {
  size_t n_tag_sets; /* Number of unique tag sets that matched view. */
  const census_view_aggregation_data *data; /* n_tag_sets entries */
} census_view_data;

CENSUSAPI const census_view_data *census_view_get_data(const census_view *view);

CENSUSAPI void census_view_reset(census_view *view);

#ifdef __cplusplus
}
#endif

#endif /* CENSUS_CENSUS_H */