application-example.conf

# The app secret must be set for Play to start, even though Becquerel code doesn't use it.
# https://www.playframework.com/documentation/2.5.x/ApplicationSecret
# "changeme" will only work in development mode.
play.crypto.secret="changeme"
# For production, use some long random value that you generate yourself.
# The rest of this example assumes that secrets are provided as environment variables beginning with VAULT_.
play.crypto.secret=${?VAULT_APPLICATION_SECRET}

# Register Becquerel components for dependency injection. This is required; leave it as is.
# https://www.playframework.com/documentation/2.5.x/ScalaDependencyInjection#Play-libraries
play.modules.enabled += "com.thumbtack.becquerel.modules.BecquerelServiceModule"
# Becquerel's reporting module only supports InfluxDB out of the box. You may want to extend it or add your own.
play.modules.enabled += "com.thumbtack.becquerel.modules.MetricsReportingModule"

# Register filters for logging, metrics, and response compression. This is required; leave it as is.
# https://www.playframework.com/documentation/2.5.x/ScalaHttpFilters#Using-filters
play.http.filters = "com.thumbtack.becquerel.filters.Filters"

# Adapt Play's default executor to mostly-synchronous operation.
# Olingo, JDBC, and the current BQ client don't have any async capabilities.
# https://www.playframework.com/documentation/2.5.x/ThreadPools#Highly-synchronous
# You can tune the pool size based on available cores, DB connections, etc. in your environment.
# The `MDCPropagatingDispatcherConfigurator` part is required to propagate run IDs for logging.
akka {
  actor {
    default-dispatcher {
      type = "com.thumbtack.becquerel.util.MDCPropagatingDispatcherConfigurator"
      executor = "thread-pool-executor"
      throughput = 1
      thread-pool-executor {
        fixed-pool-size = 50
      }
    }
  }
}

# Thread pool for periodically fetching metadata (table schemas, etc.) from data sources.
# https://www.playframework.com/documentation/2.5.x/ThreadPools#Using-other-thread-pools
# This configuration is required, but, like the Play executor, can be adapted to your environment.
contexts {
  metadataRefresh {
    executor = "thread-pool-executor"
    throughput = 1
    thread-pool-executor {
      fixed-pool-size = 20
    }
  }
}

# This is an optional workaround for incorrect X-Forwarded-Port and X-Forwarded-Proto headers.
# It can be set to true or false to force generated URLs to use https: or http: schemes respectively,
# or omitted to trust the headers when generating URLs.
#reverseRouteSecureOverride = true

# Glob expressions for config that shouldn't be displayed in full on the status page.
# Can take single globs or glob lists.
censor {
  env = [ "VAULT_*", "top_secret" ]
  javaProp = "play.crypto.*"
}

# If using JDBC data sources, configure one or more named JDBC connection pools here.
# https://www.playframework.com/documentation/2.5.x/ScalaDatabase#configuring-jdbc-connection-pools
db {
  default {
    driver = "org.postgresql.Driver"
    url = "jdbc:postgresql://localhost/ds2"
  }
}

# OData services with different kinds of datasource.
services {
  # This is the service's short name. It's used to build service-related URLs.
  bq {
    # This is a BigQuery-backed service.
    type = "com.thumbtack.becquerel.datasources.bigquery.BqService"

    # Becquerel currently only supports using a single service account per OData service:
    # https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount
    # Credentials may be supplied in JSON format as a string, possibly from an environment variable:
    credentialsJSON = ${?VAULT_SERVICE_ACCOUNT_CREDENTIALS}
    # or as a path to a file containing credentials in the same JSON format.
    credentials = "bq-credentials.json"
    # If both are provided, `credentialsJSON` will be used first if set, then the `credentials` file.
    # If no credentials are provided, the service will use application default credentials:
    # https://developers.google.com/identity/protocols/application-default-credentials
    # This configuration is useful if running on Google Compute Engine,
    # or if you're testing on a personal account and you've run the Google Cloud SDK's `gcloud init`.

    # Projects to scan for BigQuery datasets.
    # If not provided, will default to whatever project your service account credentials came from.
    projects = [ "tt-becquerel" ]

    # Optional. If provided, will only search for tables within these datasets.
    datasets = [ "dvdstore" ]

    # Optional. If provided, will only publish tables that match these glob expressions.
    tables = [ "*" ]

    # OData entity collection names are built from a table's project, dataset, and table IDs.
    # If you have only one project, you can set this flag for shorter names with no project ID.
    omitProjectID = true

    # Scala duration expression. Refresh cached schemas for BigQuery tables this often.
    metadataRefreshInterval = "10m"

    # If you have multiple Becquerel instances, you may want to prevent them from fetching metadata all at once,
    # especially immediately after a deployment, since this might result in a rapidly exhausted API quota.
    # Setting this to a nonzero duration will cause Becquerel to wait for a random fraction of the duration
    # before fetching metadata for the first time.
    metadataRefreshDelayMax = "0m"
  }

  pg {
    # This is a JDBC-backed service.
    type = "com.thumbtack.becquerel.datasources.jdbc.JdbcService"

    # Name of a Play DB connection pool, defined above.
    playDBName = "default"

    # Analogous to BqService's omitProjectID option.
    # JDBC tables are identified by catalog, schema, and table name, but many DBs only support one catalog per connection;
    # for example, Postgres and H2. Additionally, many applications will only ever use one schema.
    # Set either of these flags to shorten OData entity collection names.
    omitCatalogID = true
    omitSchemaID = true

    # Fetch only certain types of table.
    # Depending on your database and driver, JDBC table metadata APIs may report things like sequences,
    # system tables, and other kinds of object that you don't care about and that Becquerel can't query.
    # These settings are safe for Postgres but you may need to experiment for your database.
    safeTableTypes = [ "TABLE", "VIEW" ]
  }

  es {
    # This is an Elasticsearch-backed service.
    type = "com.thumbtack.becquerel.datasources.elasticsearch.EsService"

    # `elasticsearch://` URL containing one or more `host:port` combinations.
    # Note that Becquerel only supports versions 5.x and possibly newer, and only over the HTTP protocol.
    url = "elasticsearch://localhost:9200"

    # List of indexes to include. Can take ES glob patterns, including "*" or "_all" to list all available aliases.
    indexes = [ "dvdstore__*" ]

    # List of aliases to include. Can take ES glob patterns, including "*" or "_all" to list all available aliases.
    # Will automatically include all indexes for each alias; they won't be listed separately from the alias.
    aliases = [ ]

    # You must provide at least one of `indexes` or `aliases` for anything to show up in this service.

    # Retry configuration defaults. You can leave this section out entirely to use these.
    retry {
      # Time the ES client will wait before retrying the first failed request.
      # Wait time will double with every subsequent failure, up to maxWait.
      initialWait = "1s"

      # Maximum time the ES client will wait before retrying a failed request.
      maxWait = "8s"

      # The ES client will attempt an ES command this many times.
      # Set to 1 to disable retries.
      maxAttempts = 5

      # HTTP status codes that will trigger retry behavior.
      # Default includes explicit rate limiting as well as server/upstream timeouts.
      statusCodes = [ 429, 502, 503, 504 ]
    }
  }

  shadow {
    # This is a shadow service, a meta-service backed by two other services.
    type = "com.thumbtack.becquerel.ShadowService"

    # The primary service always provides the metadata.
    primary = "bq"

    # The function of the secondary service may vary by table.
    secondary = "es"

    # Scala duration after which queries to the secondary service will be abandoned.
    # May be omitted or set to `Inf` to wait forever.
    secondaryTimeout = "5s"

    # List of globs matching OData entity collection names on either service.
    # For shadowed tables, this service will always query both the primary and secondary backing services,
    # and always return data solely from the primary service. If the secondary query is successful,
    # its result will be compared for equivalence to the result from the primary, and the comparision will be logged.
    # This mode is mostly useful for testing when migrating between different data sources that should contain the same data.
    shadowTables = [ ]

    # List of globs matching OData entity collection names on either service.
    # For fallback tables, this service will query the secondary backing service first and return that result.
    # If it fails with some sort of error, or if `secondaryTimeout` is set and it times out,
    # this service will then query the primary backing service and return its result.
    # This mode is useful for when a faster but less capable or less reliable service (the secondary)
    # and a slower but more powerful service (the primary) are used in tandem and contain the same data;
    # one example is Elasticsearch falling back to BigQuery if Elasticsearch is down.
    fallbackTables = [ "dvdstore__*" ]

    # For tables not in either list, only the primary service will be used.
  }
}

# Config for metrics reporters.
metrics {
  # Maps to various InfluxDB client parameters. See `com.thumbtack.becquerel.util.InfluxDBReporterImpl`.
  # Omit the `influxdb` section to disable InfluxDB metrics forwarding.
  influxdb {
    interval = "1m"
    senderType = "HTTP"
    protocol = "https"
    host = "influxdb.example.com"
    port = 8086
    database = "becquerel"
    username = "becquerel"
    password = ${?VAULT_INFLUXDB_PASSWORD}
    hostnameTag = "hostname"
    tags {
      environment = "example"
    }
    fields {
      meters = [ "count", "m1_rate" ]
      timers = [ "count", "min", "max", "mean", "p50", "p99", "m1_rate" ]
    }
  }
}