HTTP/HTTPS Protocol Implementation
HTTP Internals, Raw Client/Server, Keep-Alive, SSL Wrapping
Introduction
Every web API, browser request, and microservice call runs over HTTP. Understanding the protocol at the byte level, what is actually sent over the wire, makes you a better security engineer and API designer. This lesson implements HTTP from scratch using raw sockets: a client that speaks HTTP/1.1 and a minimal server that parses and responds to requests, then wraps everything with TLS.
HTTP Request & Response Anatomy
HTTP Request
POST /api/users HTTP/1.1
Host: api.example.com
Content-Type: application/json
Authorization: Bearer eyJhbGci...
Content-Length: 42
Connection: keep-alive
{"name": "Alice", "email": "a@b.com"}- Request line: METHOD path HTTP-version
- Headers: Key: Value, one per line
- Blank line: \r\n marks header end
- Body: Optional (POST/PUT)
HTTP Response
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 67
X-Request-ID: abc-123
Connection: keep-alive
{"id": 1, "name": "Alice", "created_at": "2024-01-15"}- Status line: HTTP-version status-code reason
- Headers: Server metadata
- Blank line: Separates headers from body
- Body: HTML, JSON, binary data
Python: Raw HTTP Client
Connect & Build Request
# raw_http_client.py
import socket
def build_http_request(
method: str,
host: str,
path: str,
headers: dict | None = None,
body: str | None = None
) -> bytes:
"""Build a raw HTTP/1.1 request as bytes."""
body_bytes = body.encode("utf-8") if body else b""
base_headers = {
"Host": host,
"Connection": "close",
"User-Agent": "PythonRawHTTP/1.0",
}
if body_bytes:
base_headers["Content-Length"] = str(len(body_bytes))
base_headers["Content-Type"] = "application/json"
if headers:
base_headers.update(headers)
# Build request line + headers
request_lines = [f"{method} {path} HTTP/1.1"]
for key, value in base_headers.items():
request_lines.append(f"{key}: {value}")
# Headers end with a blank line (\r\n\r\n)
request = "\r\n".join(request_lines) + "\r\n\r\n"
return request.encode("utf-8") + body_bytes
# Print what the request looks like on the wire
request = build_http_request("GET", "httpbin.org", "/get")
print("Raw HTTP request bytes:")
print(repr(request[:150]))
print()
print("Formatted:")
print(request.decode())Expected Output:
Raw HTTP request bytes: b'GET /get HTTP/1.1\r\nHost: httpbin.org\r\nConnection: close\r\nUser-Agent: PythonRawHTTP/1.0\r\n\r\n' Formatted: GET /get HTTP/1.1 Host: httpbin.org Connection: close User-Agent: PythonRawHTTP/1.0
Send Request & Parse Response
# raw_http_send.py, continued
import socket
def send_raw_http(host: str, port: int, request: bytes) -> tuple[dict, str]:
"""Send a raw HTTP request and parse the response."""
with socket.create_connection((host, port), timeout=10) as sock:
sock.sendall(request)
# Receive the full response
response_bytes = b""
while True:
chunk = sock.recv(4096)
if not chunk:
break
response_bytes += chunk
# Split headers and body at \r\n\r\n
header_section, _, body = response_bytes.partition(b"\r\n\r\n")
header_lines = header_section.decode("utf-8").split("\r\n")
# Parse status line
status_line = header_lines[0]
print(f"Status: {status_line}")
# Parse headers into dict
headers = {}
for line in header_lines[1:]:
if ": " in line:
key, _, value = line.partition(": ")
headers[key.lower()] = value
return headers, body.decode("utf-8")
# Make a GET request
request = build_http_request("GET", "httpbin.org", "/get")
headers, body = send_raw_http("httpbin.org", 80, request)
print(f"Content-Type: {headers.get('content-type')}")
print(f"Body (first 200 chars):\n{body[:200]}")Expected Output:
Status: HTTP/1.1 200 OK
Content-Type: application/json
Body (first 200 chars):
{
"args": {},
"headers": {
"Host": "httpbin.org",
"User-Agent": "PythonRawHTTP/1.0",
"X-Amzn-Trace-Id": "Root=1-69963d90-4dddceeb42c964ea5bbfc6d0"
},
"origin": "73.200.240.240",
"url": "http://httpbin.org/get"
}Minimal HTTP Server
# minimal_http_server.py
import socket
import json
from datetime import datetime, timezone
HOST = "127.0.0.1"
PORT = 8080
ROUTES = {
"/": ("200 OK", "text/plain", "Welcome to minimal HTTP server!"),
"/health": ("200 OK", "application/json",
json.dumps({"status": "ok", "version": "1.0"})),
}
def parse_request(raw: bytes) -> dict:
"""Parse an HTTP request into method, path, headers, body."""
text, _, body = raw.partition(b"\r\n\r\n")
lines = text.decode("utf-8").split("\r\n")
method, path, _ = lines[0].split(" ", 2)
headers = {}
for line in lines[1:]:
if ": " in line:
k, _, v = line.partition(": ")
headers[k.lower()] = v
return {"method": method, "path": path, "headers": headers, "body": body}
def build_response(status: str, content_type: str, body: str) -> bytes:
body_bytes = body.encode("utf-8")
response = (
f"HTTP/1.1 {status}\r\n"
f"Content-Type: {content_type}\r\n"
f"Content-Length: {len(body_bytes)}\r\n"
f"Date: {datetime.now(timezone.utc).strftime('%a, %d %b %Y %H:%M:%S GMT')}\r\n"
f"\r\n"
)
return response.encode("utf-8") + body_bytes
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind((HOST, PORT))
server.listen(5)
print(f"Server at http://{HOST}:{PORT}")
while True:
conn, addr = server.accept()
with conn:
raw = conn.recv(4096)
if not raw:
continue
req = parse_request(raw)
print(f"{req['method']} {req['path']}")
if req["path"] in ROUTES:
status, ct, body = ROUTES[req["path"]]
else:
status, ct, body = "404 Not Found", "text/plain", "Not found"
conn.sendall(build_response(status, ct, body))Expected Output:
Server at http://127.0.0.1:8080 GET / GET /health GET /missing
HTTPS, Wrapping with SSL
HTTPS is HTTP over TLS. Python's ssl module wraps any socket with TLS, performing the handshake automatically.
# https_client.py
import socket
import ssl
def https_get(host: str, path: str) -> str:
"""Make an HTTPS GET request using ssl.wrap_socket."""
# Create SSL context with certificate verification
context = ssl.create_default_context()
with socket.create_connection((host, 443), timeout=10) as raw_sock:
# Wrap the socket with TLS
with context.wrap_socket(raw_sock, server_hostname=host) as tls_sock:
# Inspect the TLS connection
cert = tls_sock.getpeercert()
cipher = tls_sock.cipher()
print(f"TLS version: {tls_sock.version()}")
print(f"Cipher: {cipher[0]}")
print(f"Server CN: {cert['subject'][0][0][1]}")
# Send HTTP request over TLS
request = (
f"GET {path} HTTP/1.1\r\n"
f"Host: {host}\r\n"
f"Connection: close\r\n"
f"\r\n"
).encode()
tls_sock.sendall(request)
# Receive response
response = b""
while chunk := tls_sock.recv(4096):
response += chunk
header_section, _, body = response.partition(b"\r\n\r\n")
status = header_section.split(b"\r\n")[0].decode()
return status, body.decode()[:200]
status, body = https_get("httpbin.org", "/get")
print(f"\nStatus: {status}")
print(f"Body preview: {body}")Expected Output:
TLS version: TLSv1.2
Cipher: ECDHE-RSA-AES128-GCM-SHA256
Server CN: httpbin.org
Status: HTTP/1.1 200 OK
Body preview: {
"args": {},
"headers": {
"Host": "httpbin.org",
"X-Amzn-Trace-Id": "Root=1-69963ef8-3995cf605950aa5758bbb21d"
},
"origin": "73.200.240.240",
"url": "https://httpbin.org/get"
}Key Takeaways
- HTTP is plain text over TCP, request line, headers, blank line, optional body; knowing this helps debug any web issue
- Headers control behavior, Content-Type, Authorization, Connection, Content-Length all affect how requests are processed
- HTTPS = HTTP + TLS, the
sslmodule wraps any socket transparently - Always verify certificates, use
ssl.create_default_context()which enables verification by default - TLS 1.3 is the current standard, older versions (1.0, 1.1) are disabled in modern systems
- Raw HTTP knowledge, enables you to build custom clients, proxies, protocol testers, and understand every framework
What's Next?
Lesson 8 moves beyond HTTP to custom protocol design, when you need binary efficiency, you design your own wire format.
- Binary protocol design, struct-based header + payload pattern
- Protocol Buffers, Google's efficient serialization format
- MessagePack, binary JSON alternative
- gRPC, high-performance RPC framework