PostgreSQL:
Basic to Expert

# Install PostgreSQL 16
sudo apt install postgresql postgresql-contrib

# Start service
sudo systemctl start postgresql

# Connect as postgres superuser
sudo -u postgres psql

# Create a DB and user
CREATE USER vignesh WITH PASSWORD 'securepass';
CREATE DATABASE mydb OWNER vignesh;

# Connect from terminal
psql -h localhost -U vignesh -d mydb

-- Schema organization
CREATE SCHEMA banking;

-- Table with modern best practices
CREATE TABLE banking.accounts (
  id          BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
  account_no  TEXT        NOT NULL UNIQUE,
  customer_id BIGINT      NOT NULL,
  balance     NUMERIC(15,2) NOT NULL DEFAULT 0,
  status      TEXT        NOT NULL DEFAULT 'ACTIVE',
  created_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
  updated_at  TIMESTAMPTZ
);

-- Alter table
ALTER TABLE banking.accounts ADD COLUMN branch_code TEXT;
ALTER TABLE banking.accounts ALTER COLUMN status SET NOT NULL;
ALTER TABLE banking.accounts RENAME TO bank_accounts;

-- Drop with safety
DROP TABLE IF EXISTS banking.accounts CASCADE;

-- INSERT with RETURNING
INSERT INTO accounts (account_no, customer_id, balance)
VALUES ('AC001', 42, 10000.00)
RETURNING id, created_at;

-- Bulk INSERT
INSERT INTO accounts (account_no, customer_id)
SELECT gen_random_uuid()::TEXT, id FROM customers;

-- UPSERT (INSERT ON CONFLICT)
INSERT INTO accounts (account_no, balance)
VALUES ('AC001', 5000)
ON CONFLICT (account_no) DO UPDATE
  SET balance = EXCLUDED.balance;

-- UPDATE with JOIN using FROM clause
UPDATE accounts a
SET balance = a.balance + t.amount
FROM transactions t
WHERE t.account_id = a.id AND t.status = 'PENDING'
RETURNING a.id, a.balance;

-- DELETE with RETURNING
DELETE FROM accounts
WHERE status = 'CLOSED' AND balance = 0
RETURNING id;

SELECT
  a.account_no,
  c.name AS customer_name,
  a.balance,
  CASE
    WHEN a.balance >= 100000 THEN 'PREMIUM'
    WHEN a.balance >= 10000  THEN 'REGULAR'
    ELSE 'BASIC'
  END AS tier,
  TO_CHAR(a.created_at, 'DD-Mon-YYYY') AS opened_on
FROM       accounts a
INNER JOIN customers c ON c.id = a.customer_id
WHERE  a.status = 'ACTIVE'
  AND  a.balance > 0
ORDER BY a.balance DESC
LIMIT 20 OFFSET 0;

-- INNER JOIN: rows matching in both
SELECT a.id, t.amount
FROM accounts a
INNER JOIN transactions t ON t.account_id = a.id;

-- LEFT JOIN: all accounts, with or without transactions
SELECT a.id, COUNT(t.id) AS txn_count
FROM accounts a
LEFT JOIN transactions t ON t.account_id = a.id
GROUP BY a.id;

-- FULL OUTER JOIN: all rows from both sides
SELECT a.id, b.id AS branch_id
FROM accounts a
FULL OUTER JOIN branches b ON b.code = a.branch_code;

-- CROSS JOIN: cartesian product
SELECT p.name, c.currency
FROM products p CROSS JOIN currencies c;

-- SELF JOIN: employee hierarchy
SELECT e.name, m.name AS manager
FROM employees e
LEFT JOIN employees m ON m.id = e.manager_id;

-- LATERAL JOIN: correlated subquery per row
SELECT c.name, recent.amount
FROM customers c,
LATERAL (
  SELECT amount FROM transactions
  WHERE customer_id = c.id
  ORDER BY created_at DESC LIMIT 1
) recent;

-- Standard aggregates
SELECT
  branch_code,
  COUNT(*)            AS account_count,
  SUM(balance)         AS total_balance,
  AVG(balance)::NUMERIC(12,2) AS avg_balance,
  MIN(balance)         AS min_bal,
  MAX(balance)         AS max_bal,
  STDDEV(balance)      AS std_dev
FROM accounts
WHERE status = 'ACTIVE'
GROUP BY branch_code
HAVING SUM(balance) > 1000000
ORDER BY total_balance DESC;

-- GROUPING SETS / ROLLUP / CUBE
SELECT branch_code, status, COUNT(*)
FROM accounts
GROUP BY ROLLUP(branch_code, status);

-- Filter within aggregate (pg9.4+)
SELECT
  COUNT(*) FILTER (WHERE status = 'ACTIVE')  AS active_count,
  COUNT(*) FILTER (WHERE status = 'CLOSED')  AS closed_count
FROM accounts;

CREATE TABLE transactions (
  id          BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
  account_id  BIGINT      NOT NULL
              REFERENCES accounts(id) ON DELETE RESTRICT,
  amount      NUMERIC(15,2) NOT NULL
              CHECK (amount <> 0),
  type        TEXT        NOT NULL
              CHECK (type IN ('CREDIT','DEBIT')),
  ref_no      TEXT        UNIQUE,
  created_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
  CONSTRAINT chk_future_date CHECK (created_at <= now() + '1 day'::INTERVAL)
);

-- Deferrable constraint (evaluated at commit not statement)
ALTER TABLE transactions
  ADD CONSTRAINT fk_account
  FOREIGN KEY (account_id) REFERENCES accounts(id)
  DEFERRABLE INITIALLY DEFERRED;

-- Standard B-tree
CREATE INDEX idx_accounts_customer ON accounts(customer_id);

-- Composite index (column order matters!)
CREATE INDEX idx_txn_account_date ON transactions(account_id, created_at DESC);

-- Partial index — only active accounts
CREATE INDEX idx_active_accounts ON accounts(customer_id)
  WHERE status = 'ACTIVE';

-- Expression index — case insensitive email search
CREATE INDEX idx_email_lower ON customers(LOWER(email));

-- GIN index for JSONB
CREATE INDEX idx_meta_gin ON accounts USING GIN(metadata);

-- Concurrent — no table lock!
CREATE INDEX CONCURRENTLY idx_txn_status ON transactions(status);

-- Check index usage
SELECT indexname, idx_scan, idx_tup_read
FROM  pg_stat_user_indexes
WHERE schemaname = 'public'
ORDER BY idx_scan;

-- Basic transaction
BEGIN;
  UPDATE accounts SET balance = balance - 1000 WHERE id = 1;
  UPDATE accounts SET balance = balance + 1000 WHERE id = 2;
COMMIT;  -- or ROLLBACK

-- SAVEPOINT for partial rollback
BEGIN;
  INSERT INTO audit_log VALUES ('started');
  SAVEPOINT sp1;
  UPDATE accounts SET balance = -999 WHERE id = 1;  -- risky
  ROLLBACK TO sp1;  -- undo just the UPDATE
  RELEASE SAVEPOINT sp1;
COMMIT;

-- Isolation levels
BEGIN TRANSACTION ISOLATION LEVEL SERIALIZABLE;
BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;
BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;  -- default

SELECT
  account_no,
  branch_code,
  balance,

  -- Ranking
  ROW_NUMBER() OVER (ORDER BY balance DESC)           AS overall_rank,
  RANK()       OVER (PARTITION BY branch_code ORDER BY balance DESC) AS branch_rank,
  DENSE_RANK() OVER (PARTITION BY branch_code ORDER BY balance DESC) AS dense,
  NTILE(4)     OVER (ORDER BY balance)                AS quartile,
  PERCENT_RANK() OVER (ORDER BY balance)              AS pct_rank,

  -- Running total
  SUM(balance) OVER (PARTITION BY branch_code
                     ORDER BY created_at
                     ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
                  AS running_balance,

  -- Lag/Lead: compare to prev/next row
  LAG(balance, 1)  OVER (PARTITION BY branch_code ORDER BY created_at) AS prev_bal,
  LEAD(balance, 1) OVER (PARTITION BY branch_code ORDER BY created_at) AS next_bal,

  -- First/Last value in partition
  FIRST_VALUE(balance) OVER (PARTITION BY branch_code ORDER BY created_at) AS first_bal

FROM accounts;

-- Standard CTE (named subquery)
WITH high_value AS (
  SELECT id, customer_id, balance
  FROM  accounts
  WHERE balance > 500000
),
customer_totals AS (
  SELECT customer_id, SUM(balance) AS total
  FROM  high_value
  GROUP BY customer_id
)
SELECT c.name, ct.total
FROM  customer_totals ct
JOIN  customers c ON c.id = ct.customer_id
ORDER BY ct.total DESC;

-- Recursive CTE: org chart / hierarchy
WITH RECURSIVE emp_tree AS (
  -- anchor: top-level manager
  SELECT id, name, manager_id, 1 AS depth
  FROM  employees
  WHERE manager_id IS NULL

  UNION ALL

  -- recursive: join to parent
  SELECT e.id, e.name, e.manager_id, t.depth + 1
  FROM  employees e
  JOIN  emp_tree t ON t.id = e.manager_id
)
SELECT REPEAT('  ', depth - 1) || name AS org_chart, depth
FROM  emp_tree
ORDER BY depth, name;

-- Create with JSONB column
ALTER TABLE accounts ADD COLUMN metadata JSONB;

-- Insert JSON
UPDATE accounts SET metadata = '{"kyc": true, "tags": ["premium","nri"], "risk": 2}'
WHERE id = 1;

-- Extract operators
SELECT
  metadata->'kyc'           AS kyc_json,    -- returns JSON
  metadata->>'kyc'          AS kyc_text,    -- returns TEXT
  metadata#>>'{tags,0}'     AS first_tag,  -- nested path as text
  metadata->'tags'          AS tags_array
FROM accounts;

-- Containment
SELECT * FROM accounts
WHERE metadata @> '{"kyc": true}';    -- contains

-- Update nested key (jsonb_set)
UPDATE accounts
SET metadata = jsonb_set(metadata, '{risk}', '3')
WHERE id = 1;

-- GIN index on JSONB
CREATE INDEX idx_meta_gin ON accounts USING GIN(metadata);

-- Function: transfer funds between accounts
CREATE OR REPLACE FUNCTION transfer_funds(
  p_from_id   BIGINT,
  p_to_id     BIGINT,
  p_amount    NUMERIC
) RETURNS VOID LANGUAGE plpgsql AS $$
DECLARE
  v_balance NUMERIC;
BEGIN
  -- Lock source row
  SELECT balance INTO v_balance
  FROM accounts WHERE id = p_from_id FOR UPDATE;

  IF v_balance < p_amount THEN
    RAISE EXCEPTION 'Insufficient funds: balance=%, amount=%',
                    v_balance, p_amount;
  END IF;

  UPDATE accounts SET balance = balance - p_amount WHERE id = p_from_id;
  UPDATE accounts SET balance = balance + p_amount WHERE id = p_to_id;

  INSERT INTO audit_log(from_id, to_id, amount)
  VALUES(p_from_id, p_to_id, p_amount);
END;
$$;

-- Call it
SELECT transfer_funds(1, 2, 500.00);

-- Auto-update updated_at on any change
CREATE OR REPLACE FUNCTION set_updated_at()
RETURNS TRIGGER LANGUAGE plpgsql AS $$
BEGIN
  NEW.updated_at := now();
  RETURN NEW;
END;
$$;

CREATE TRIGGER trg_accounts_updated_at
  BEFORE UPDATE ON accounts
  FOR EACH ROW EXECUTE FUNCTION set_updated_at();

-- Parent table (no storage itself)
CREATE TABLE transactions_part (
  id         BIGINT NOT NULL,
  account_id BIGINT NOT NULL,
  amount     NUMERIC(15,2),
  txn_date   DATE NOT NULL
) PARTITION BY RANGE (txn_date);

-- Partitions by quarter
CREATE TABLE txn_2024_q1 PARTITION OF transactions_part
  FOR VALUES FROM ('2024-01-01') TO ('2024-04-01');

CREATE TABLE txn_2024_q2 PARTITION OF transactions_part
  FOR VALUES FROM ('2024-04-01') TO ('2024-07-01');

-- Default partition catches everything else
CREATE TABLE txn_default PARTITION OF transactions_part DEFAULT;

-- Postgres auto-prunes: this only scans txn_2024_q1
SELECT * FROM transactions_part
WHERE txn_date BETWEEN '2024-01-01' AND '2024-03-31';

-- Detach old partition (instant, no data movement)
ALTER TABLE transactions_part DETACH PARTITION txn_2024_q1;

-- Basic plan (estimated costs only)
EXPLAIN
SELECT * FROM transactions WHERE account_id = 42;

-- Full analysis with actual timings
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT * FROM transactions WHERE account_id = 42;

-- Reading output example:
-- Index Scan using idx_account on transactions  (cost=0.56..8.58 rows=1 width=80)
--                                                      ↑            ↑
--                                               startup cost      total cost
-- Index Cond: (account_id = 42)
-- Buffers: shared hit=3                ← all from cache, 0 disk reads
-- Planning Time: 0.1 ms
-- Execution Time: 0.05 ms

-- Enable pg_stat_statements extension first
CREATE EXTENSION pg_stat_statements;

-- Top 10 slowest queries
SELECT
  LEFT(query, 80)               AS query,
  calls,
  ROUND(total_exec_time::NUMERIC, 2) AS total_ms,
  ROUND(mean_exec_time::NUMERIC, 2)  AS avg_ms,
  rows
FROM  pg_stat_statements
ORDER BY mean_exec_time DESC
LIMIT 10;

-- Table bloat check
SELECT
  schemaname, tablename,
  pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size,
  n_dead_tup, n_live_tup,
  ROUND(100*n_dead_tup/NULLIF(n_live_tup+n_dead_tup,0),1) AS dead_pct
FROM pg_stat_user_tables
ORDER BY n_dead_tup DESC;

-- On PRIMARY: postgresql.conf
wal_level = replica
max_wal_senders = 5
wal_keep_size = '1GB'

-- Create replication user
CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'replpass';

-- pg_hba.conf on primary
host replication replicator 10.0.0.0/24 scram-sha-256

-- On STANDBY: take base backup
pg_basebackup -h primary_host -U replicator -D /var/lib/postgresql/data -P -R

-- -R auto-creates standby.signal + postgresql.auto.conf with:
-- primary_conninfo = 'host=primary_host user=replicator ...'

-- Check replication lag on primary
SELECT client_addr, state, sent_lsn, replay_lsn,
  (sent_lsn - replay_lsn) AS lag_bytes
FROM pg_stat_replication;

-- Manual vacuum
VACUUM accounts;              -- reclaim dead tuples
VACUUM ANALYZE accounts;     -- + update statistics
VACUUM FULL accounts;        -- rewrite table, exclusive lock!
VACUUM FREEZE accounts;      -- freeze old XIDs to prevent wraparound

-- Monitor autovacuum
SELECT
  schemaname, relname,
  last_autovacuum,
  last_autoanalyze,
  n_dead_tup,
  n_ins_since_vacuum
FROM pg_stat_user_tables
ORDER BY n_dead_tup DESC;

-- XID wraparound risk (danger if > 1.5B)
SELECT datname, age(datfrozenxid) AS xid_age
FROM  pg_database
ORDER BY xid_age DESC;

-- Use pg_repack (no lock, online table repack)
-- pg_repack -d mydb -t accounts

-- pg_trgm: fast fuzzy search
CREATE EXTENSION pg_trgm;
CREATE INDEX idx_name_trgm ON customers USING GIN(name gin_trgm_ops);
SELECT * FROM customers WHERE name ILIKE '%vign%';  -- uses index!

-- pgvector: AI semantic search
CREATE EXTENSION vector;
ALTER TABLE documents ADD COLUMN embedding vector(1536);
CREATE INDEX ON documents USING hnsw(embedding vector_cosine_ops);
SELECT id, content
FROM  documents
ORDER BY embedding <-> '[0.1,0.2,...]'::vector
LIMIT 5;

-- Create role hierarchy
CREATE ROLE app_readonly;
CREATE ROLE app_readwrite;
CREATE ROLE app_admin;

GRANT USAGE ON SCHEMA banking TO app_readonly;
GRANT SELECT ON ALL TABLES IN SCHEMA banking TO app_readonly;

GRANT app_readonly TO app_readwrite;
GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA banking TO app_readwrite;

-- Application user inherits from role
CREATE USER api_user WITH PASSWORD 'xxx';
GRANT app_readwrite TO api_user;

-- Row Level Security: users only see their own accounts
ALTER TABLE accounts ENABLE ROW LEVEL SECURITY;

CREATE POLICY own_accounts ON accounts
  FOR ALL
  USING (customer_id = current_setting('app.current_user_id')::BIGINT);

-- App sets context before queries
SET app.current_user_id = '42';

-- Active queries with lock info
SELECT pid, now() - query_start AS duration, state, wait_event, query
FROM  pg_stat_activity
WHERE state <> 'idle'
ORDER BY duration DESC;

-- Lock waits: who's blocking whom
SELECT
  blocking.pid AS blocked_by,
  blocking.query AS blocking_query,
  blocked.pid   AS blocked_pid,
  blocked.query AS blocked_query
FROM  pg_stat_activity blocked
JOIN  pg_stat_activity blocking
  ON  blocking.pid = ANY(pg_blocking_pids(blocked.pid));

-- Terminate a long-running query safely
SELECT pg_cancel_backend(12345);    -- sends SIGINT (graceful)
SELECT pg_terminate_backend(12345); -- sends SIGTERM (force)

-- Table sizes with toast & indexes
SELECT
  relname,
  pg_size_pretty(pg_relation_size(oid))            AS table_size,
  pg_size_pretty(pg_indexes_size(oid))             AS index_size,
  pg_size_pretty(pg_total_relation_size(oid))       AS total_size
FROM  pg_class
WHERE relkind = 'r' AND relnamespace = 'public'::regnamespace
ORDER BY pg_total_relation_size(oid) DESC;