History

Protect deployment database connections (#2057)

Rhys Sullivan committed 1 month ago

Commit 25a490d1 parent 2e5aa167 files · +286 −2Browse files

Showing the changes to .github (2 of 7 files). Show all

Unified · Split

2 files
  1. .github/scripts/check-database-capacity.py +59 −0
  2. .github/workflows/database-capacity.yml +27 −0
.github/scripts/check-database-capacity.py+59−0
@@ -0,0 +1,59 @@
1+"""Read aggregate connection capacity with libpq; never print credentials or SQL data."""
2+
3+import json
4+import os
5+import subprocess
6+import sys
7+from urllib.parse import unquote, urlparse
8+
9+
10+def main() -> int:
11+ try:
12+ url = urlparse(os.environ["DATABASE_URL"])
13+ if url.scheme not in ("postgres", "postgresql") or not url.hostname:
14+ raise ValueError("Invalid database URL")
15+ if url.hostname.endswith(".psdb.cloud") and url.port not in (None, 5432):
16+ raise ValueError("Capacity checks require the direct endpoint")
17+ env = {
18+ **os.environ,
19+ "PGHOST": url.hostname,
20+ "PGPORT": str(url.port or 5432),
21+ "PGUSER": unquote(url.username or ""),
22+ "PGPASSWORD": unquote(url.password or ""),
23+ "PGDATABASE": unquote(url.path.removeprefix("/")),
24+ "PGSSLMODE": "require",
25+ "PGCONNECT_TIMEOUT": "10",
26+ "PGAPPNAME": "database-capacity-check",
27+ "PGOPTIONS": "-c default_transaction_read_only=on -c statement_timeout=10000",
28+ }
29+ result = subprocess.run(
30+ ["psql", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", """
31+ SELECT json_build_object(
32+ 'limit', current_setting('max_connections')::int,
33+ 'reserved', current_setting('superuser_reserved_connections')::int
34+ + current_setting('reserved_connections')::int,
35+ 'used', count(*)::int
36+ ) FROM pg_stat_activity WHERE backend_type = 'client backend'
37+ """],
38+ env=env,
39+ capture_output=True,
40+ text=True,
41+ timeout=25,
42+ check=True,
43+ )
44+ capacity = json.loads(result.stdout)
45+ if any(type(capacity[key]) is not int for key in ("limit", "reserved", "used")):
46+ raise ValueError("Invalid capacity response")
47+ free = capacity["limit"] - capacity["reserved"] - capacity["used"]
48+ print(json.dumps({**capacity, "ordinary_free": free, "minimum_free": 10}))
49+ if free < 10:
50+ print("::error::Database connection headroom is below 10 slots. Inspect direct clients and the PgBouncer budget.")
51+ return 1
52+ return 0
53+ except (KeyError, ValueError, OSError, subprocess.SubprocessError):
54+ print("::error::Database capacity check failed. Check direct endpoint access and provider health.")
55+ return 1
56+
57+
58+if __name__ == "__main__":
59+ sys.exit(main())
.github/workflows/database-capacity.yml+27−0
@@ -0,0 +1,27 @@
1+name: Database capacity
2+
3+on:
4+ schedule:
5+ - cron: "2-57/5 * * * *"
6+ workflow_dispatch:
7+
8+permissions:
9+ contents: read
10+
11+concurrency:
12+ group: database-capacity
13+ cancel-in-progress: false
14+
15+jobs:
16+ check:
17+ runs-on: ubuntu-24.04
18+ timeout-minutes: 2
19+ environment: production
20+ steps:
21+ - uses: actions/checkout@v4
22+ # psql is supplied by the Ubuntu runner image. A failed check uses the
23+ # repository's Actions failure notifications; no customer data is logged.
24+ - name: Check ordinary connection headroom
25+ run: python3 .github/scripts/check-database-capacity.py
26+ env:
27+ DATABASE_URL: ${{ secrets.DATABASE_URL }}