Rollback rsyslog to 8.27.0 to prevent memory leak.

build/cmake/rsyslog.cmake

@@ -71,8 +71,8 @@ endif()
 list(APPEND RSYSLOG_CONFIGURE_CMD env)
 list(APPEND RSYSLOG_CONFIGURE_CMD LIBESTR_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include)
 list(APPEND RSYSLOG_CONFIGURE_CMD "LIBESTR_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lestr")
-list(APPEND RSYSLOG_CONFIGURE_CMD LIBFASTJSON_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include/libfastjson)
-list(APPEND RSYSLOG_CONFIGURE_CMD "LIBFASTJSON_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lfastjson")
+list(APPEND RSYSLOG_CONFIGURE_CMD JSON_C_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include/libfastjson)
+list(APPEND RSYSLOG_CONFIGURE_CMD "JSON_C_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lfastjson")
 list(APPEND RSYSLOG_CONFIGURE_CMD LIBLOGGING_STDLOG_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include)
 list(APPEND RSYSLOG_CONFIGURE_CMD "LIBLOGGING_STDLOG_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -llogging-stdlog")
 if(ENABLE_HADOOP_ANALYTICS)

build/cmake/versions.cmake

@@ -96,8 +96,10 @@ set(RUBY_VERSION 2.3.4)
 set(RUBY_HASH cd9808bb53824d6edb58beaadd3906cb23b987438ce75ab7bb279b2229930e2f)
 set(RUBYGEMS_VERSION 2.6.12)
 set(RUBYGEMS_HASH 2b724ce280daa4d0a0a209fe44c2bbe01dacb78b95301b35f8a6d40ce3ff05b6)
-set(RSYSLOG_VERSION 8.28.0)
-set(RSYSLOG_HASH 4ca5405908d612d45da700e36856430510875518eb8028d296d1ee4d2c44678e)
+# Hold at 8.27.0 until memory leak is fixed:
+# https://github.com/rsyslog/rsyslog/issues/1668
+set(RSYSLOG_VERSION 8.27.0)
+set(RSYSLOG_HASH 02aefbba59324a6d8b70036a67686bed5f0c7be4ced62c039af6ee694cb5b1fd)
 set(RUNIT_VERSION 2.1.2)
 set(RUNIT_HASH 6c985fbfe3a34608eb3c53dc719172c4)
 set(SHELLCHECK_VERSION 0.4.6)

test/processes/test_rsyslog.rb

@@ -0,0 +1,84 @@
+require_relative "../test_helper"
+
+class Test::Processes::TestRsyslog < Minitest::Test
+  include ApiUmbrellaTestHelpers::Setup
+  include ApiUmbrellaTestHelpers::Logging
+
+  def setup
+    super
+    setup_server
+  end
+
+  # To make sure rsyslog doesn't leak memory while logging requests:
+  # https://github.com/rsyslog/rsyslog/issues/1668
+  def test_memory_leak
+    # Make some initial requests, to ensure rsyslog is warmed up, which should
+    # stabilize its memory use.
+    make_requests(1000)
+    warmed_memory_use = memory_use
+    warmed_error_log_size = elasticsearch_error_log_size
+
+    # Make more requests.
+    make_requests(5000)
+    final_memory_use = memory_use
+    final_error_log_size = elasticsearch_error_log_size
+
+    # Compare the memory use before and after making requests. We're going to
+    # assume it should be not growing, or within 2% of the starting point.
+    if(final_memory_use.fetch(:vsz) > warmed_memory_use.fetch(:vsz))
+      assert_in_epsilon(warmed_memory_use.fetch(:vsz), final_memory_use.fetch(:vsz), 0.02)
+    end
+    if(final_memory_use.fetch(:rss) > warmed_memory_use.fetch(:rss))
+      assert_in_epsilon(warmed_memory_use.fetch(:rss), final_memory_use.fetch(:rss), 0.02)
+    end
+
+    # Also ensure nothing was generated in the elasticsearch error log file,
+    # since the specific problem in v8.28.0 generated error data.
+    assert_equal(warmed_error_log_size, final_error_log_size)
+  end
+
+  private
+
+  def make_requests(count)
+    request = nil
+    hydra = Typhoeus::Hydra.new
+    count.times do
+      request = Typhoeus::Request.new("http://127.0.0.1:9080/api/hello/", log_http_options)
+      hydra.queue(request)
+    end
+    hydra.run
+
+    # Just check for the last request made and make sure it gets logged.
+    assert_response_code(200, request.response)
+    wait_for_log(request.response)
+  end
+
+  def memory_use
+    pid = File.read(File.join($config["run_dir"], "rsyslogd.pid"))
+    output, status = run_shell("ps -o vsz=,rss= #{pid}")
+    assert_equal(0, status, output)
+
+    columns = output.strip.split(/\s+/)
+    assert_equal(2, columns.length, columns)
+
+    memory = {
+      :vsz => Integer(columns[0]),
+      :rss => Integer(columns[1]),
+    }
+
+    assert_operator(memory[:vsz], :>, 0)
+    assert_operator(memory[:rss], :>, 0)
+
+    memory
+  end
+
+  def elasticsearch_error_log_size
+    size = 0
+    path = File.join($config["log_dir"], "rsyslog/elasticsearch_error.log")
+    if(File.exist?(path))
+      size = File.size(path)
+    end
+
+    size
+  end
+end