HTTP/HTTPS Protocol Implementation

HTTP Internals, Raw Client/Server, Keep-Alive, SSL Wrapping

Introduction

Every web API, browser request, and microservice call runs over HTTP. Understanding the protocol at the byte level, what is actually sent over the wire, makes you a better security engineer and API designer. This lesson implements HTTP from scratch using raw sockets: a client that speaks HTTP/1.1 and a minimal server that parses and responds to requests, then wraps everything with TLS.

HTTP Request & Response Anatomy

HTTP Request
POST /api/users HTTP/1.1
Host: api.example.com
Content-Type: application/json
Authorization: Bearer eyJhbGci...
Content-Length: 42
Connection: keep-alive

{"name": "Alice", "email": "a@b.com"}
  • Request line: METHOD path HTTP-version
  • Headers: Key: Value, one per line
  • Blank line: \r\n marks header end
  • Body: Optional (POST/PUT)
HTTP Response
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 67
X-Request-ID: abc-123
Connection: keep-alive

{"id": 1, "name": "Alice", "created_at": "2024-01-15"}
  • Status line: HTTP-version status-code reason
  • Headers: Server metadata
  • Blank line: Separates headers from body
  • Body: HTML, JSON, binary data

Python: Raw HTTP Client

Connect & Build Request

# raw_http_client.py
import socket

def build_http_request(
    method: str,
    host: str,
    path: str,
    headers: dict | None = None,
    body: str | None = None
) -> bytes:
    """Build a raw HTTP/1.1 request as bytes."""

    body_bytes = body.encode("utf-8") if body else b""

    base_headers = {
        "Host": host,
        "Connection": "close",
        "User-Agent": "PythonRawHTTP/1.0",
    }

    if body_bytes:
        base_headers["Content-Length"] = str(len(body_bytes))
        base_headers["Content-Type"] = "application/json"

    if headers:
        base_headers.update(headers)

    # Build request line + headers
    request_lines = [f"{method} {path} HTTP/1.1"]
    for key, value in base_headers.items():
        request_lines.append(f"{key}: {value}")

    # Headers end with a blank line (\r\n\r\n)
    request = "\r\n".join(request_lines) + "\r\n\r\n"
    return request.encode("utf-8") + body_bytes

# Print what the request looks like on the wire
request = build_http_request("GET", "httpbin.org", "/get")
print("Raw HTTP request bytes:")
print(repr(request[:150]))
print()
print("Formatted:")
print(request.decode())
Expected Output:
Raw HTTP request bytes:
b'GET /get HTTP/1.1\r\nHost: httpbin.org\r\nConnection: close\r\nUser-Agent: PythonRawHTTP/1.0\r\n\r\n'

Formatted:
GET /get HTTP/1.1
Host: httpbin.org
Connection: close
User-Agent: PythonRawHTTP/1.0

Send Request & Parse Response

# raw_http_send.py, continued
import socket

def send_raw_http(host: str, port: int, request: bytes) -> tuple[dict, str]:
    """Send a raw HTTP request and parse the response."""
    with socket.create_connection((host, port), timeout=10) as sock:
        sock.sendall(request)

        # Receive the full response
        response_bytes = b""
        while True:
            chunk = sock.recv(4096)
            if not chunk:
                break
            response_bytes += chunk

    # Split headers and body at \r\n\r\n
    header_section, _, body = response_bytes.partition(b"\r\n\r\n")
    header_lines = header_section.decode("utf-8").split("\r\n")

    # Parse status line
    status_line = header_lines[0]
    print(f"Status: {status_line}")

    # Parse headers into dict
    headers = {}
    for line in header_lines[1:]:
        if ": " in line:
            key, _, value = line.partition(": ")
            headers[key.lower()] = value

    return headers, body.decode("utf-8")

# Make a GET request
request = build_http_request("GET", "httpbin.org", "/get")
headers, body = send_raw_http("httpbin.org", 80, request)

print(f"Content-Type: {headers.get('content-type')}")
print(f"Body (first 200 chars):\n{body[:200]}")
Expected Output:
Status: HTTP/1.1 200 OK
Content-Type: application/json
Body (first 200 chars):
{
  "args": {},
  "headers": {
    "Host": "httpbin.org",
    "User-Agent": "PythonRawHTTP/1.0",
    "X-Amzn-Trace-Id": "Root=1-69963d90-4dddceeb42c964ea5bbfc6d0"
  },
  "origin": "73.200.240.240",
  "url": "http://httpbin.org/get"
}

Minimal HTTP Server

# minimal_http_server.py
import socket
import json
from datetime import datetime, timezone

HOST = "127.0.0.1"
PORT = 8080

ROUTES = {
    "/": ("200 OK", "text/plain", "Welcome to minimal HTTP server!"),
    "/health": ("200 OK", "application/json",
                json.dumps({"status": "ok", "version": "1.0"})),
}

def parse_request(raw: bytes) -> dict:
    """Parse an HTTP request into method, path, headers, body."""
    text, _, body = raw.partition(b"\r\n\r\n")
    lines = text.decode("utf-8").split("\r\n")
    method, path, _ = lines[0].split(" ", 2)
    headers = {}
    for line in lines[1:]:
        if ": " in line:
            k, _, v = line.partition(": ")
            headers[k.lower()] = v
    return {"method": method, "path": path, "headers": headers, "body": body}

def build_response(status: str, content_type: str, body: str) -> bytes:
    body_bytes = body.encode("utf-8")
    response = (
        f"HTTP/1.1 {status}\r\n"
        f"Content-Type: {content_type}\r\n"
        f"Content-Length: {len(body_bytes)}\r\n"
        f"Date: {datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S GMT')}\r\n"
        f"\r\n"
    )
    return response.encode("utf-8") + body_bytes

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
    server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server.bind((HOST, PORT))
    server.listen(5)
    print(f"Server at http://{HOST}:{PORT}")

    while True:
        conn, addr = server.accept()
        with conn:
            raw = conn.recv(4096)
            if not raw:
                continue
            req = parse_request(raw)
            print(f"{req['method']} {req['path']}")

            if req["path"] in ROUTES:
                status, ct, body = ROUTES[req["path"]]
            else:
                status, ct, body = "404 Not Found", "text/plain", "Not found"

            conn.sendall(build_response(status, ct, body))
Expected Output:
Server at http://127.0.0.1:8080
GET /
GET /health
GET /missing

HTTPS, Wrapping with SSL

HTTPS is HTTP over TLS. Python's ssl module wraps any socket with TLS, performing the handshake automatically.

# https_client.py
import socket
import ssl

def https_get(host: str, path: str) -> str:
    """Make an HTTPS GET request using ssl.wrap_socket."""
    # Create SSL context with certificate verification
    context = ssl.create_default_context()

    with socket.create_connection((host, 443), timeout=10) as raw_sock:
        # Wrap the socket with TLS
        with context.wrap_socket(raw_sock, server_hostname=host) as tls_sock:
            # Inspect the TLS connection
            cert = tls_sock.getpeercert()
            cipher = tls_sock.cipher()
            print(f"TLS version: {tls_sock.version()}")
            print(f"Cipher: {cipher[0]}")
            print(f"Server CN: {cert['subject'][0][0][1]}")

            # Send HTTP request over TLS
            request = (
                f"GET {path} HTTP/1.1\r\n"
                f"Host: {host}\r\n"
                f"Connection: close\r\n"
                f"\r\n"
            ).encode()
            tls_sock.sendall(request)

            # Receive response
            response = b""
            while chunk := tls_sock.recv(4096):
                response += chunk

    header_section, _, body = response.partition(b"\r\n\r\n")
    status = header_section.split(b"\r\n")[0].decode()
    return status, body.decode()[:200]

status, body = https_get("httpbin.org", "/get")
print(f"\nStatus: {status}")
print(f"Body preview: {body}")
Expected Output:
TLS version: TLSv1.2
Cipher: ECDHE-RSA-AES128-GCM-SHA256
Server CN: httpbin.org

Status: HTTP/1.1 200 OK
Body preview: {
  "args": {},
  "headers": {
    "Host": "httpbin.org",
    "X-Amzn-Trace-Id": "Root=1-69963ef8-3995cf605950aa5758bbb21d"
  },
  "origin": "73.200.240.240",
  "url": "https://httpbin.org/get"
}

Key Takeaways

  • HTTP is plain text over TCP, request line, headers, blank line, optional body; knowing this helps debug any web issue
  • Headers control behavior, Content-Type, Authorization, Connection, Content-Length all affect how requests are processed
  • HTTPS = HTTP + TLS, the ssl module wraps any socket transparently
  • Always verify certificates, use ssl.create_default_context() which enables verification by default
  • TLS 1.3 is the current standard, older versions (1.0, 1.1) are disabled in modern systems
  • Raw HTTP knowledge, enables you to build custom clients, proxies, protocol testers, and understand every framework
What's Next?

Lesson 8 moves beyond HTTP to custom protocol design, when you need binary efficiency, you design your own wire format.

  • Binary protocol design, struct-based header + payload pattern
  • Protocol Buffers, Google's efficient serialization format
  • MessagePack, binary JSON alternative
  • gRPC, high-performance RPC framework