Add CircleCI config to build, test and release (#47)

## Which problem is this PR solving?
- Adds CircleCI configuration to build and test rdslogs and manage the release process to Github. Also includes updating the docker image used to create the binaries to golang 1.18 which includes an updated OpenSSL to resolve CVE (modified from the one in honeymarker).
- Adds a default configuration file (ancient builds for a single platform used to just generate this on the fly, but that's trickier in a multi-platform world, so we just generated it locally and committed it).
- Update the build_pkg.sh script (modified from the one in honeymarker).
- Add a build_docker.sh script (from the one in honeymarker).

- Closes #41
- Closes #46

## Short description of the changes
- Add CircleCI configuration to build, test and release

.circleci/config.yml

@@ -0,0 +1,217 @@
+version: 2.1
+
+orbs:
+  aws-cli: circleci/[email protected]
+  docker: circleci/[email protected]
+
+executors:
+  pkg:
+    ## executor with fpm and npmbuild
+    docker:
+      - image: alanfranz/fpm-within-docker:centos-8
+
+platform_matrix: &platform_matrix
+  matrix:
+    parameters:
+      os: &oses ["linux", "darwin"]
+      arch: &arches ["amd64", "arm64", "arm", "386"]
+    exclude:
+      - os: darwin
+        arch: arm
+      - os: darwin
+        arch: "386"
+
+jobs:
+  test:
+    docker:
+      - image: cimg/go:1.18
+    steps:
+      - checkout
+      - run: go test --timeout 10s -v ./...
+
+  build_bins:
+    docker:
+      - image: cimg/go:1.18
+    parameters:
+      os:
+        description: Target operating system
+        type: enum
+        enum: *oses
+        default: "linux"
+      arch:
+        description: Target architecture
+        type: enum
+        enum: *arches
+        default: "amd64"
+    steps:
+      - checkout
+      - run: |
+          GOOS=<< parameters.os >> \
+          GOARCH=<< parameters.arch >> \
+          CGO_ENABLED=0 \
+          go build -ldflags "-X main.BuildID=${CIRCLE_TAG}" \
+          -o ~/binaries/rdslogs-<< parameters.os >>-<< parameters.arch >> \
+          .
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - binaries/rdslogs-<< parameters.os >>-<< parameters.arch >>
+      - store_artifacts:
+          path: binaries/rdslogs-<< parameters.os >>-<< parameters.arch >>
+
+  ## We only have to build packages for linux, so we iterate architectures and build rpm and deb for each.
+  build_packages:
+    executor: pkg
+    parameters:
+      arch:
+        description: Target architecture
+        type: enum
+        enum: *arches
+        default: "amd64"
+    steps:
+      - attach_workspace:
+          at: ~/
+      - checkout
+      - run: ./build-pkg.sh -m << parameters.arch >> -v "${CIRCLE_TAG}" -t deb
+      - run: ./build-pkg.sh -m << parameters.arch >> -v "${CIRCLE_TAG}" -t rpm
+      - run: echo "finished building packages" && find ~/packages -ls
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - packages/<< parameters.arch >>/*
+      - store_artifacts:
+          path: ~/packages/<< parameters.arch >>
+
+  consolidate_artifacts:
+    docker:
+    - image: cimg/go:1.18
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run: cp -R ~/binaries ~/artifacts
+      - run: find ~/packages -type f -print0 |xargs -0 -I {} cp {} ~/artifacts
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - artifacts
+
+  publish_github:
+    docker:
+      - image: cibuilds/github:0.13.0
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            echo "about to publish to tag ${CIRCLE_TAG}"
+            ls -l ~/artifacts/*
+            ghr -draft -n ${CIRCLE_TAG} -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} ${CIRCLE_TAG} ~/artifacts
+
+  publish_s3:
+    executor: aws-cli/default
+    steps:
+      - attach_workspace:
+          at: ~/
+      - aws-cli/install
+      - aws-cli/setup:
+          aws-access-key-id: AWS_ACCESS_KEY_ID
+          aws-secret-access-key: AWS_SECRET_ACCESS_KEY
+          aws-region: AWS_REGION
+      - run:
+          name: sync_s3_artifacts
+          command: aws s3 sync ~/artifacts s3://honeycomb-builds/honeycombio/rdslogs/${CIRCLE_TAG}/
+
+  build_docker:
+    docker:
+      - image: cimg/go:1.18
+    steps:
+      - run: go install github.com/google/ko@latest
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: build docker images and publish locally
+          command: ./build-docker.sh
+
+  publish_docker:
+    docker:
+      - image: cimg/go:1.18
+    steps:
+      - run: go install github.com/google/ko@latest
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: build docker images and publish to Docker Hub
+          environment:
+            KO_DOCKER_REPO: honeycombio
+          command: |
+            echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin;
+            ./build-docker.sh
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - test:
+          filters:
+            tags:
+              only: /.*/
+      - build_bins:
+          <<: *platform_matrix
+          requires:
+            - test
+          filters:
+            tags:
+              only: /.*/
+      - build_packages:
+          matrix:
+            parameters:
+              arch: *arches
+          context: Honeycomb Secrets for Public Repos
+          requires:
+            - build_bins
+          filters:
+            tags:
+              # allow tags that start with t so we can test builds without publishing
+              only: /^[vt].*/
+            branches:
+              ignore: /.*/
+      - build_docker:
+          requires:
+            - test
+          filters:
+            tags:
+              only: /.*/
+      - consolidate_artifacts:
+          requires:
+            - build_packages
+          filters:
+            tags:
+              only: /.*/
+      - publish_github:
+          context: Honeycomb Secrets for Public Repos
+          requires:
+            - consolidate_artifacts
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/
+      - publish_s3:
+          context: Honeycomb Secrets for Public Repos
+          requires:
+            - consolidate_artifacts
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/
+      - publish_docker:
+          context: Honeycomb Secrets for Public Repos
+          requires:
+            - build_docker
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

RELEASING.md

@@ -0,0 +1,8 @@
+# Releasing Process
+
+1. Add release entry to [changelog](./CHANGELOG.md)
+3. Open a PR with the above, and merge that into main
+4. Create new tag on merged commit with the new version (e.g. `v0.2.1`)
+5. Push the tag upstream (this will kick off the release pipeline in CI)
+6. Copy change log entry for newest version into draft GitHub release created as part of CI publish steps
+7. Update [public docs](https://github.com/honeycombio/docs/blob/main/scripts)

build-docker.sh

@@ -0,0 +1,24 @@
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+TAGS="latest"
+VERSION="dev"
+if [[ -n ${CIRCLE_TAG:-} ]]; then
+    # trim 'v' prefix if present
+    VERSION=${CIRCLE_TAG#"v"}
+    # append version to image tags
+    TAGS+=",$VERSION"
+fi
+
+unset GOOS
+unset GOARCH
+export KO_DOCKER_REPO=${KO_DOCKER_REPO:-ko.local}
+export GOFLAGS="-ldflags=-X=main.BuildID=$VERSION"
+export SOURCE_DATE_EPOCH=$(date +%s)
+# shellcheck disable=SC2086
+ko publish \
+  --tags "${TAGS}" \
+  --base-import-paths \
+  --platform "linux/amd64,linux/arm64" \
+  .

build-pkg.sh

@@ -4,32 +4,38 @@
 set -e
 
 function usage() {
-    echo "Usage: build-pkg.sh -v <version> -t <package_type>"
+    echo "Usage: build-pkg.sh -v <version> -t <package_type> -m arch"
     exit 2
 }
 
-while getopts "v:t:" opt; do
+while getopts "v:t:m:" opt; do
     case "$opt" in
     v)
         version=$OPTARG
         ;;
     t)
         pkg_type=$OPTARG
         ;;
+    m)
+        arch=$OPTARG
+        ;;
     esac
 done
 
-if [ -z "$version" ] || [ -z "$pkg_type" ]; then
+if [ -z "$version" ] || [ -z "$pkg_type" ] || [ -z "$arch" ]; then
     usage
 fi
 
+PACKAGE_DIR=~/packages/${arch}
+mkdir -p ${PACKAGE_DIR}
 fpm -s dir -n rdslogs \
-    -m "Honeycomb <team@honeycomb.io>" \
-    -p $GOPATH/bin \
+    -m "Honeycomb <solutions@honeycomb.io>" \
+    -p ${PACKAGE_DIR} \
     -v $version \
     -t $pkg_type \
+    -a $arch \
     --pre-install=./preinstall \
-    $GOPATH/bin/rdslogs=/usr/bin/rdslogs \
+    ~/binaries/rdslogs-linux-${arch}=/usr/bin/rdslogs \
     ./rdslogs.upstart=/etc/init/rdslogs.conf \
     ./rdslogs.service=/lib/systemd/system/rdslogs.service \
     ./rdslogs.conf=/etc/rdslogs/rdslogs.conf \

rdslogs.conf

@@ -0,0 +1,58 @@
+[Application Options]
+; AWS region to use
+; Region = us-east-1
+
+; RDS instance identifier
+; InstanceIdentifier =
+
+; RDS database type. Accepted values are mysql and postgresql.
+; DBType = mysql
+
+; Log file type. Accepted values are query and audit. Audit is currently only supported for mysql.
+; LogType = query
+
+; RDS log file to retrieve
+; LogFile =
+
+; Download old logs instead of tailing the current log
+; Download = false
+
+; directory in to which log files are downloaded
+; DownloadDir = ./
+
+; number of lines to request at a time from AWS. Larger number will be more efficient, smaller number will allow for longer lines
+; NumLines = 10000
+
+; how many seconds to pause when rate limited by AWS.
+; BackoffTimer = 5
+
+; output for the logs: stdout or honeycomb
+; Output = stdout
+
+; Team write key, when output is honeycomb
+; WriteKey =
+
+; Name of the dataset, when output is honeycomb
+; Dataset =
+
+; Hostname for the Honeycomb API server
+; APIHost = https://api.honeycomb.io/
+
+; Replaces the query field with a one-way hash of the contents
+; ScrubQuery = false
+
+; Only send 1 / N log lines
+; SampleRate = 1
+
+; Extra fields to send in request, in the style of "field:value"
+; AddFields =
+
+; Number of parsers to spin up. Currently only supported for the mysql parser.
+; NumParsers = 4
+
+; Output the current version and exit
+; Version = false
+
+; turn on debugging output
+; Debug = false
+