History

Protect deployment database connections (#2057)

Rhys Sullivan committed 1 month ago

Commit 25a490d1 parent 2e5aa167 files · +286 −2Browse files

Showing the changes to .github (2 of 7 files). Show all

Unified · Split

2 files
  1. .github/scripts/check-database-capacity.py +59 −0
  2. .github/workflows/database-capacity.yml +27 −0
.github/scripts/check-database-capacity.py+59−0
@@ -0,0 +1,59 @@
1"""Read aggregate connection capacity with libpq; never print credentials or SQL data."""
2
3import json
4import os
5import subprocess
6import sys
7from urllib.parse import unquote, urlparse
8
9
10def main() -> int:
11 try:
12 url = urlparse(os.environ["DATABASE_URL"])
13 if url.scheme not in ("postgres", "postgresql") or not url.hostname:
14 raise ValueError("Invalid database URL")
15 if url.hostname.endswith(".psdb.cloud") and url.port not in (None, 5432):
16 raise ValueError("Capacity checks require the direct endpoint")
17 env = {
18 **os.environ,
19 "PGHOST": url.hostname,
20 "PGPORT": str(url.port or 5432),
21 "PGUSER": unquote(url.username or ""),
22 "PGPASSWORD": unquote(url.password or ""),
23 "PGDATABASE": unquote(url.path.removeprefix("/")),
24 "PGSSLMODE": "require",
25 "PGCONNECT_TIMEOUT": "10",
26 "PGAPPNAME": "database-capacity-check",
27 "PGOPTIONS": "-c default_transaction_read_only=on -c statement_timeout=10000",
28 }
29 result = subprocess.run(
30 ["psql", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", """
31 SELECT json_build_object(
32 'limit', current_setting('max_connections')::int,
33 'reserved', current_setting('superuser_reserved_connections')::int
34 + current_setting('reserved_connections')::int,
35 'used', count(*)::int
36 ) FROM pg_stat_activity WHERE backend_type = 'client backend'
37 """],
38 env=env,
39 capture_output=True,
40 text=True,
41 timeout=25,
42 check=True,
43 )
44 capacity = json.loads(result.stdout)
45 if any(type(capacity[key]) is not int for key in ("limit", "reserved", "used")):
46 raise ValueError("Invalid capacity response")
47 free = capacity["limit"] - capacity["reserved"] - capacity["used"]
48 print(json.dumps({**capacity, "ordinary_free": free, "minimum_free": 10}))
49 if free < 10:
50 print("::error::Database connection headroom is below 10 slots. Inspect direct clients and the PgBouncer budget.")
51 return 1
52 return 0
53 except (KeyError, ValueError, OSError, subprocess.SubprocessError):
54 print("::error::Database capacity check failed. Check direct endpoint access and provider health.")
55 return 1
56
57
58if __name__ == "__main__":
59 sys.exit(main())
.github/workflows/database-capacity.yml+27−0
@@ -0,0 +1,27 @@
1name: Database capacity
2
3on:
4 schedule:
5 - cron: "2-57/5 * * * *"
6 workflow_dispatch:
7
8permissions:
9 contents: read
10
11concurrency:
12 group: database-capacity
13 cancel-in-progress: false
14
15jobs:
16 check:
17 runs-on: ubuntu-24.04
18 timeout-minutes: 2
19 environment: production
20 steps:
21 - uses: actions/checkout@v4
22 # psql is supplied by the Ubuntu runner image. A failed check uses the
23 # repository's Actions failure notifications; no customer data is logged.
24 - name: Check ordinary connection headroom
25 run: python3 .github/scripts/check-database-capacity.py
26 env:
27 DATABASE_URL: ${{ secrets.DATABASE_URL }}