From ae706a560ef3032e12f2037486990890138f189c Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Wed, 7 Aug 2024 16:34:10 -0500 Subject: [PATCH 1/2] chore: remove oldOpenresty in favor of newer nginx - Add echo module to nginx - Remove unused valgrind --- .gitignore | 1 + docs/contributing.md | 22 ---- nix/nginx/conf/nginx.conf | 2 +- nix/nginxScript.nix | 13 ++- nix/pgValgrindScript.nix | 33 ------ nix/valgrind.supp | 216 -------------------------------------- shell.nix | 18 +--- 7 files changed, 15 insertions(+), 290 deletions(-) delete mode 100644 nix/pgValgrindScript.nix delete mode 100644 nix/valgrind.supp diff --git a/.gitignore b/.gitignore index 531952f..6821a0d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ nix/nginx/logs/nginx.pid *.bc *.control *.so +nginx.pid diff --git a/docs/contributing.md b/docs/contributing.md index 801e073..3bfa862 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -114,28 +114,6 @@ nixops destroy -d pg_net --confirm nixops delete -d pg_net ``` -### Run Valgrind to check memory leaks - -Start a postgres instance under valgrind. - -``` -valgrind-net-with-pg-12 - -# it should show this output: -# Connect to this db from another shell using the following command: -# -# psql -h -U postgres - -# connect to the db in another shell -psql -h -U postgres - -create extension pg_net; -select net.http_get('https://supabase.io'); - -# stop the db instance -# then see the valgrind results in "valgrindlog" -``` - ## Documentation All public API must be documented. Building documentation requires python 3.6+ diff --git a/nix/nginx/conf/nginx.conf b/nix/nginx/conf/nginx.conf index 804044b..10070a0 100644 --- a/nix/nginx/conf/nginx.conf +++ b/nix/nginx/conf/nginx.conf @@ -1,5 +1,5 @@ daemon off; -error_log /dev/stdout info; +pid ./nginx.pid; events {} diff --git a/nix/nginxScript.nix b/nix/nginxScript.nix index 4d9599d..df977b4 100644 --- a/nix/nginxScript.nix +++ b/nix/nginxScript.nix @@ -1,14 +1,19 @@ -{ openresty, writeShellScriptBin } : +{ nginx, nginxModules, writeShellScriptBin } : let + customNginx = nginx.override { + modules = [ + nginxModules.echo + ]; + }; script = '' set -euo pipefail - export PATH=${openresty}/bin:"$PATH" + export PATH=${customNginx}/bin:"$PATH" - trap 'killall openresty' sigint sigterm exit + trap 'killall nginx' sigint sigterm exit - openresty -p nix/nginx & + nginx -p nix/nginx -e stderr & "$@" ''; diff --git a/nix/pgValgrindScript.nix b/nix/pgValgrindScript.nix deleted file mode 100644 index 4c4bc19..0000000 --- a/nix/pgValgrindScript.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ postgresql, valgrind, writeShellScriptBin } : - -let - LOGMIN = builtins.getEnv "LOGMIN"; - ver = builtins.head (builtins.splitVersion postgresql.version); - valgrindLogFile = "valgrindlog"; - script = '' - set -euo pipefail - - export PATH=${postgresql}/bin:"$PATH" - - tmpdir="$(mktemp -d)" - - export PGDATA="$tmpdir" - export PGHOST="$tmpdir" - export PGUSER=postgres - export PGDATABASE=postgres - - PGTZ=UTC initdb --no-locale --encoding=UTF8 --nosync -U "$PGUSER" - - rm ${valgrindLogFile} - - echo -e "Connect to this db from another shell using the following command: \n" - echo -e "\t psql -h "$tmpdir" -U postgres \n" - - ${valgrind}/bin/valgrind --log-file=${valgrindLogFile} \ - --leak-check=yes --trace-children=yes --track-origins=yes --suppressions=${./valgrind.supp} \ - postgres -c shared_preload_libraries="pg_net" -c listen_addresses="" -k $PGDATA - - rm -rf "$tmpdir" - ''; -in -writeShellScriptBin "valgrind-net-with-pg-${ver}" script diff --git a/nix/valgrind.supp b/nix/valgrind.supp deleted file mode 100644 index d9e2128..0000000 --- a/nix/valgrind.supp +++ /dev/null @@ -1,216 +0,0 @@ -# Taken from https://github.com/postgres/postgres/blob/master/src/tools/valgrind.supp -# -# This is a suppression file for use with Valgrind tools. File format -# documentation: -# http://valgrind.org/docs/manual/mc-manual.html#mc-manual.suppfiles - -# The libc symbol that implements a particular standard interface is -# implementation-dependent. For example, strncpy() shows up as "__GI_strncpy" -# on some platforms. Use wildcards to avoid mentioning such specific names. -# Avoid mentioning functions that are good candidates for inlining, -# particularly single-caller static functions. Suppressions mentioning them -# would be ineffective at higher optimization levels. - - -# We have occasion to write raw binary structures to disk or to the network. -# These may contain uninitialized padding bytes. Since recipients also ignore -# those bytes as padding, this is harmless. - -{ - padding_pgstat_send - Memcheck:Param - socketcall.send(msg) - - fun:*send* - fun:pgstat_send -} - -{ - padding_pgstat_sendto - Memcheck:Param - socketcall.sendto(msg) - - fun:*send* - fun:pgstat_send -} - -{ - padding_pgstat_write - Memcheck:Param - write(buf) - - ... - fun:pgstat_write_statsfiles -} - -{ - padding_XLogRecData_CRC - Memcheck:Value8 - - fun:pg_comp_crc32c - fun:XLogRecordAssemble -} - -{ - padding_XLogRecData_write - Memcheck:Param - pwrite64(buf) - - ... - fun:XLogWrite -} - -{ - padding_relcache - Memcheck:Param - write(buf) - - ... - fun:write_relcache_init_file -} - -{ - padding_reorderbuffer_serialize - Memcheck:Param - write(buf) - - ... - fun:ReorderBufferSerializeTXN -} - -{ - padding_twophase_prepare - Memcheck:Param - write(buf) - - ... - fun:EndPrepare -} - - -{ - padding_twophase_CRC - Memcheck:Value8 - fun:pg_comp_crc32c - fun:EndPrepare -} - -{ - padding_bootstrap_initial_xlog_write - Memcheck:Param - write(buf) - - ... - fun:BootStrapXLOG -} - -{ - padding_bootstrap_control_file_write - Memcheck:Param - write(buf) - - ... - fun:WriteControlFile - fun:BootStrapXLOG -} - -{ - bootstrap_write_relmap_overlap - Memcheck:Overlap - fun:memcpy* - fun:write_relmap_file - fun:RelationMapFinishBootstrap -} - - -# gcc on ppc64 can generate a four-byte read to fetch the final "char" fields -# of a FormData_pg_cast. This is valid compiler behavior, because a proper -# FormData_pg_cast has trailing padding. Tuples we treat as structures omit -# that padding, so Valgrind reports an invalid read. Practical trouble would -# entail the missing pad bytes falling in a different memory page. So long as -# the structure is aligned, that will not happen. -{ - overread_tuplestruct_pg_cast - Memcheck:Addr4 - - fun:IsBinaryCoercible -} - -# Atomic writes to 64bit atomic vars uses compare/exchange to -# guarantee atomic writes of 64bit variables. pg_atomic_write is used -# during initialization of the atomic variable; that leads to an -# initial read of the old, undefined, memory value. But that's just to -# make sure the swap works correctly. -{ - uninitialized_atomic_init_u64 - Memcheck:Cond - fun:pg_atomic_exchange_u64_impl - fun:pg_atomic_write_u64_impl - fun:pg_atomic_init_u64_impl -} - - -# Python's allocator does some low-level tricks for efficiency. Those -# can be disabled for better instrumentation; but few people testing -# postgres will have such a build of python. So add broad -# suppressions of the resulting errors. -# See also https://svn.python.org/projects/python/trunk/Misc/README.valgrind -{ - python_clever_allocator - Memcheck:Addr4 - fun:PyObject_Free -} - -{ - python_clever_allocator - Memcheck:Addr8 - fun:PyObject_Free -} - -{ - python_clever_allocator - Memcheck:Value4 - fun:PyObject_Free -} - -{ - python_clever_allocator - Memcheck:Value8 - fun:PyObject_Free -} - -{ - python_clever_allocator - Memcheck:Cond - fun:PyObject_Free -} - -{ - python_clever_allocator - Memcheck:Addr4 - fun:PyObject_Realloc -} - -{ - python_clever_allocator - Memcheck:Addr8 - fun:PyObject_Realloc -} - -{ - python_clever_allocator - Memcheck:Value4 - fun:PyObject_Realloc -} - -{ - python_clever_allocator - Memcheck:Value8 - fun:PyObject_Realloc -} - -{ - python_clever_allocator - Memcheck:Cond - fun:PyObject_Realloc -} diff --git a/shell.nix b/shell.nix index e855cd8..904d103 100644 --- a/shell.nix +++ b/shell.nix @@ -1,20 +1,13 @@ let - # newer nginx/openresty versions somehow do not support using a prefix directory(-p) - # and want to read from /var/nginx/ - oldOpenresty = (import (builtins.fetchTarball { - name = "2020-04-20"; - url = "https://github.com/NixOS/nixpkgs/archive/5272327b81ed355bbed5659b8d303cf2979b6953.tar.gz"; - sha256 = "0182ys095dfx02vl2a20j1hz92dx3mfgz2a6fhn31bqlp1wa8hlq"; - }){}).openresty; nixpkgs = builtins.fetchTarball { name = "2020-12-22"; url = "https://github.com/NixOS/nixpkgs/archive/2a058487cb7a50e7650f1657ee0151a19c59ec3b.tar.gz"; sha256 = "1h8c0mk6jlxdmjqch6ckj30pax3hqh6kwjlvp2021x3z4pdzrn9p"; }; newPkgs = (import (builtins.fetchTarball { - name = "2023-09-16"; - url = "https://github.com/NixOS/nixpkgs/archive/ae5b96f3ab6aabb60809ab78c2c99f8dd51ee678.tar.gz"; - sha256 = "11fpdcj5xrmmngq0z8gsc3axambqzvyqkfk23jn3qkx9a5x56xxk"; + name = "24.05"; + url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/24.05.tar.gz"; + sha256 = "sha256:1lr1h35prqkd1mkmzriwlpvxcb34kmhc9dnr48gkm8hh089hifmx"; }){}); in with import nixpkgs {}; mkShell { @@ -29,8 +22,7 @@ mkShell { ]; pgWithExt = { pg }: pg.withPackages (p: [ (callPackage ./nix/pg_net.nix { postgresql = pg;}) ]); extAll = map (x: callPackage ./nix/pgScript.nix { postgresql = pgWithExt { pg = x;}; }) supportedPgVersions; - valgrind-net-with-pg-12 = callPackage ./nix/pgValgrindScript.nix { postgresql = pgWithExt { pg = postgresql_12;}; }; - nginxScript = callPackage ./nix/nginxScript.nix { openresty = oldOpenresty; }; + nginxScript = callPackage ./nix/nginxScript.nix { nginx = newPkgs.nginx; nginxModules = newPkgs.nginxModules; }; pathodScript = callPackage ./nix/pathodScript.nix {}; pythonDeps = with pythonPackages; [ pytest @@ -40,14 +32,12 @@ mkShell { format = callPackage ./nix/format.nix {}; in [ - valgrind-net-with-pg-12 extAll pythonDeps nixops format.do format.doCheck nginxScript pathodScript - oldOpenresty ]; shellHook = '' export NIX_PATH="nixpkgs=${nixpkgs}:." From ba5f9e619d55d61c2c1f14a42bdf205508a02542 Mon Sep 17 00:00:00 2001 From: steve-chavez Date: Thu, 8 Aug 2024 00:46:27 -0500 Subject: [PATCH 2/2] chore: use latest stable nixpkgs globally - Remove nixops as it's incompatible with this nixpkgs version --- docs/contributing.md | 58 ------------------- nix/deploy.nix | 117 -------------------------------------- shell.nix | 29 +++++----- test/test_http_errors.py | 6 +- test/test_worker_error.py | 6 +- 5 files changed, 19 insertions(+), 197 deletions(-) delete mode 100644 nix/deploy.nix diff --git a/docs/contributing.md b/docs/contributing.md index 3bfa862..408d5ec 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -56,64 +56,6 @@ select net.http_get('http://localhost:3000/projects'); -- * Connection #0 to host localhost left intact ``` -### Reproducible setup for load testing - -This will deploy a client and server on t3a.nano. You must have `supabase-dev` setup in `.aws/credentials`. - -```bash -nixops create nix/deploy.nix -d pg_net - -nixops deploy -d pg_net -# will take a while -``` - -Then you can connect on the client instance and do requests to the server instance through `pg_net`. - -```bash -nixops ssh -d pg_net client - -psql -U postgres - -create extension pg_net; - -select net.http_get('http://server'); -# this the default welcome page of nginx on the server instance -# "server" is already included to /etc/hosts, so `curl http://server` will give the same result - -# do some load testing -select net.http_get('http://server') from generate_series(1,1000); -# run `top` on another shell(another `nixops ssh -d pg_net client`) to check the worker behavior -``` - -#### Testing a slow response - -Locally: - -```bash -net-with-pg-12 psql - -# takes some seconds to respond -select net.http_get('http://localhost:8080/slow-reply'); -``` - -With the cloud NixOps setup: - -```bash -nixops ssh -d pg_net client - -# takes some seconds to respond -select net.http_get('http://server/slow-reply'); -``` - -#### Destroying the setup - -To destroy the instances: - -```bash -nixops destroy -d pg_net --confirm -nixops delete -d pg_net -``` - ## Documentation All public API must be documented. Building documentation requires python 3.6+ diff --git a/nix/deploy.nix b/nix/deploy.nix deleted file mode 100644 index 387ec50..0000000 --- a/nix/deploy.nix +++ /dev/null @@ -1,117 +0,0 @@ -let - region = "us-east-2"; - accessKeyId = "supabase-dev"; -in { - network.description = "pg_net load testing setup"; - - resources = { - ec2KeyPairs.netKP = { inherit region accessKeyId; }; - vpc.netVpc = { - inherit region accessKeyId; - enableDnsSupport = true; - enableDnsHostnames = true; - cidrBlock = "10.0.0.0/24"; - }; - vpcSubnets.netSubnet = {resources, ...}: { - inherit region accessKeyId; - zone = "${region}a"; - vpcId = resources.vpc.netVpc; - cidrBlock = "10.0.0.0/24"; - mapPublicIpOnLaunch = true; - }; - vpcInternetGateways.netIG = { resources, ... }: { - inherit region accessKeyId; - vpcId = resources.vpc.netVpc; - }; - vpcRouteTables.netRT = { resources, ... }: { - inherit region accessKeyId; - vpcId = resources.vpc.netVpc; - }; - vpcRoutes.netIGRoute = { resources, ... }: { - inherit region accessKeyId; - routeTableId = resources.vpcRouteTables.netRT; - destinationCidrBlock = "0.0.0.0/0"; - gatewayId = resources.vpcInternetGateways.netIG; - }; - vpcRouteTableAssociations.netTblAssoc = { resources, ... }: { - inherit region accessKeyId; - subnetId = resources.vpcSubnets.netSubnet; - routeTableId = resources.vpcRouteTables.netRT; - }; - ec2SecurityGroups.netSecGroup = {resources, ...}: { - inherit region accessKeyId; - vpcId = resources.vpc.netVpc; - rules = [ - { fromPort = 80; toPort = 80; sourceIp = "0.0.0.0/0"; } - { fromPort = 22; toPort = 22; sourceIp = "0.0.0.0/0"; } - { fromPort = 0; toPort = 65535; sourceIp = resources.vpcSubnets.netSubnet.cidrBlock; } - ]; - }; - }; - - server = { config, pkgs, resources, ... }: { - deployment = { - targetEnv = "ec2"; - ec2 = { - inherit region accessKeyId; - instanceType = "t3a.nano"; - associatePublicIpAddress = true; - keyPair = resources.ec2KeyPairs.netKP; - subnetId = resources.vpcSubnets.netSubnet; - securityGroupIds = [resources.ec2SecurityGroups.netSecGroup.name]; - }; - }; - - services.nginx = { - enable = true; - package = pkgs.openresty; - config = '' - events {} - - http { - server { - listen 0.0.0.0:80 ; - listen [::]:80 ; - server_name localhost; - - ${builtins.readFile nginx/custom.conf} - } - } - ''; - }; - networking.firewall.allowedTCPPorts = [ 80 ]; - }; - - client = { config, pkgs, nodes, resources, ... }: { - deployment = { - targetEnv = "ec2"; - ec2 = { - inherit region accessKeyId; - instanceType = "t3a.nano"; - associatePublicIpAddress = true; - ebsInitialRootDiskSize = 6; - keyPair = resources.ec2KeyPairs.netKP; - subnetId = resources.vpcSubnets.netSubnet; - securityGroupIds = [resources.ec2SecurityGroups.netSecGroup.name]; - }; - }; - - services.postgresql = { - enable = true; - package = pkgs.postgresql_12.withPackages (p: [ - (pkgs.callPackage ./pg_net.nix { postgresql = pkgs.postgresql_12;}) - ]); - authentication = pkgs.lib.mkOverride 10 '' - local all all trust - ''; - settings = { - shared_preload_libraries = "pg_net"; - }; - }; - - networking.hosts = { - "${nodes.server.config.networking.privateIPv4}" = [ "server" ]; - }; - }; - -} diff --git a/shell.nix b/shell.nix index 904d103..3b1523a 100644 --- a/shell.nix +++ b/shell.nix @@ -1,14 +1,14 @@ let + oldNixpkgs = (import (builtins.fetchTarball { + name = "2020-04-20"; + url = "https://github.com/NixOS/nixpkgs/archive/5272327b81ed355bbed5659b8d303cf2979b6953.tar.gz"; + sha256 = "0182ys095dfx02vl2a20j1hz92dx3mfgz2a6fhn31bqlp1wa8hlq"; + }){}); nixpkgs = builtins.fetchTarball { - name = "2020-12-22"; - url = "https://github.com/NixOS/nixpkgs/archive/2a058487cb7a50e7650f1657ee0151a19c59ec3b.tar.gz"; - sha256 = "1h8c0mk6jlxdmjqch6ckj30pax3hqh6kwjlvp2021x3z4pdzrn9p"; - }; - newPkgs = (import (builtins.fetchTarball { - name = "24.05"; + name = "24.05"; # May 31 2024 url = "https://github.com/NixOS/nixpkgs/archive/refs/tags/24.05.tar.gz"; sha256 = "sha256:1lr1h35prqkd1mkmzriwlpvxcb34kmhc9dnr48gkm8hh089hifmx"; - }){}); + }; in with import nixpkgs {}; mkShell { buildInputs = @@ -16,15 +16,15 @@ mkShell { supportedPgVersions = [ postgresql_12 postgresql_13 - newPkgs.postgresql_14 - newPkgs.postgresql_15 - newPkgs.postgresql_16 + postgresql_14 + postgresql_15 + postgresql_16 ]; pgWithExt = { pg }: pg.withPackages (p: [ (callPackage ./nix/pg_net.nix { postgresql = pg;}) ]); extAll = map (x: callPackage ./nix/pgScript.nix { postgresql = pgWithExt { pg = x;}; }) supportedPgVersions; - nginxScript = callPackage ./nix/nginxScript.nix { nginx = newPkgs.nginx; nginxModules = newPkgs.nginxModules; }; - pathodScript = callPackage ./nix/pathodScript.nix {}; - pythonDeps = with pythonPackages; [ + nginxScript = callPackage ./nix/nginxScript.nix {}; + pathodScript = callPackage ./nix/pathodScript.nix { mitmproxy = oldNixpkgs.mitmproxy; }; + pythonDeps = with python3Packages; [ pytest psycopg2 sqlalchemy @@ -34,14 +34,11 @@ mkShell { [ extAll pythonDeps - nixops format.do format.doCheck nginxScript pathodScript ]; shellHook = '' - export NIX_PATH="nixpkgs=${nixpkgs}:." - export NIXOPS_STATE=".deployment.nixops" export HISTFILE=.history ''; } diff --git a/test/test_http_errors.py b/test/test_http_errors.py index 0ad54b9..8a7429c 100644 --- a/test/test_http_errors.py +++ b/test/test_http_errors.py @@ -12,7 +12,8 @@ def test_get_bad_url(sess): select net.http_get('localhost:8888'); """ )) - assert "URL using bad/illegal format or missing URL" in str(execinfo) + assert r"Couldn\'t resolve proxy name" in str(execinfo) + def test_bad_post(sess): """net.http_post with an empty url + body returns an error""" @@ -23,7 +24,8 @@ def test_bad_post(sess): select net.http_post(null, '{"hello": "world"}'); """ )) - assert "violates not-null constraint" in str(execinfo) + assert 'null value in column "url"' in str(execinfo) + def test_it_keeps_working_after_many_connection_refused(sess): """the worker doesn't crash on many failed responses with connection refused""" diff --git a/test/test_worker_error.py b/test/test_worker_error.py index 0d39449..64be7f7 100644 --- a/test/test_worker_error.py +++ b/test/test_worker_error.py @@ -5,11 +5,9 @@ def test_success_when_worker_is_up(sess): """net.check_worker_is_up should not return anything when the worker is running""" - (result,) = sess.execute( - """ + (result,) = sess.execute(text(""" select net.check_worker_is_up(); - """ - ).fetchone() + """)).fetchone() assert result is not None assert result == ''