HTTP Requests

Learn how to communicate with web APIs using Python's requests library.

What is requests?

requests is the most popular library in Python for sending HTTP requests. With its simple and intuitive API, you can easily communicate with web servers.

Key Features

• Simple and intuitive API
• Automatic JSON encoding/decoding
• Session management and cookie handling
• File upload support
• Configurable timeouts

Installation

pip install requests

Basic GET Request

Let's start with the most basic GET request.

import requests

# 기본 GET 요청
response = requests.get('https://api.github.com')

# 응답 확인
print(response.status_code)  # 200
print(response.text)         # 응답 본문
print(response.json())       # JSON 형식으로 파싱

Adding Query Parameters

# URL에 직접 작성
response = requests.get('https://api.github.com/search/repositories?q=python')

# params 딕셔너리 사용 (권장)
params = {
    'q': 'python',
    'sort': 'stars',
    'order': 'desc'
}
response = requests.get('https://api.github.com/search/repositories', params=params)

print(response.url)  # 실제 요청된 URL 확인

POST Requests

Use POST requests when sending data to the server.

# Form 데이터 전송
data = {
    'username': 'john',
    'password': 'secret123'
}
response = requests.post('https://httpbin.org/post', data=data)

# JSON 데이터 전송
json_data = {
    'name': 'John Doe',
    'email': 'john@example.com',
    'age': 30
}
response = requests.post('https://httpbin.org/post', json=json_data)

print(response.json())

File Upload

# 단일 파일 업로드
files = {'file': open('report.pdf', 'rb')}
response = requests.post('https://httpbin.org/post', files=files)

# 파일명 지정
files = {
    'file': ('report.pdf', open('report.pdf', 'rb'), 'application/pdf')
}
response = requests.post('https://httpbin.org/post', files=files)

# 여러 파일 업로드
files = {
    'file1': open('image1.jpg', 'rb'),
    'file2': open('image2.jpg', 'rb')
}
response = requests.post('https://httpbin.org/post', files=files)

PUT and DELETE Requests

Use these to update or delete resources.

# PUT 요청 (전체 업데이트)
update_data = {
    'id': 1,
    'title': 'Updated Title',
    'completed': True
}
response = requests.put('https://jsonplaceholder.typicode.com/todos/1', json=update_data)

# PATCH 요청 (부분 업데이트)
partial_data = {'completed': True}
response = requests.patch('https://jsonplaceholder.typicode.com/todos/1', json=partial_data)

# DELETE 요청
response = requests.delete('https://jsonplaceholder.typicode.com/todos/1')
print(response.status_code)  # 200 또는 204

Setting Headers

You can pass additional information through request headers.

# 기본 헤더 설정
headers = {
    'User-Agent': 'MyApp/1.0',
    'Accept': 'application/json',
    'Content-Type': 'application/json'
}
response = requests.get('https://api.github.com', headers=headers)

# API 키 인증
headers = {
    'Authorization': 'Bearer YOUR_API_KEY',
    'X-API-Key': 'YOUR_API_KEY'
}
response = requests.get('https://api.example.com/data', headers=headers)

Authentication

Various authentication methods are supported.

from requests.auth import HTTPBasicAuth, HTTPDigestAuth

# Basic 인증
response = requests.get(
    'https://api.example.com/protected',
    auth=HTTPBasicAuth('username', 'password')
)

# 간단한 방법
response = requests.get(
    'https://api.example.com/protected',
    auth=('username', 'password')
)

# Bearer Token 인증
headers = {'Authorization': 'Bearer YOUR_ACCESS_TOKEN'}
response = requests.get('https://api.example.com/data', headers=headers)

# OAuth 2.0
from requests_oauthlib import OAuth2Session

oauth = OAuth2Session(client_id, token=token)
response = oauth.get('https://api.example.com/protected')

JSON Processing

You can easily handle JSON data.

# JSON 응답 파싱
response = requests.get('https://api.github.com/users/github')
data = response.json()

print(data['name'])
print(data['public_repos'])

# JSON 요청 전송
payload = {
    'name': 'John',
    'email': 'john@example.com',
    'settings': {
        'notifications': True,
        'theme': 'dark'
    }
}
response = requests.post('https://api.example.com/users', json=payload)

# 커스텀 JSON 인코더
import json

class CustomEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.isoformat()
        return super().default(obj)

response = requests.post(
    'https://api.example.com/data',
    data=json.dumps(payload, cls=CustomEncoder),
    headers={'Content-Type': 'application/json'}
)

Timeout Settings

Prevent indefinite waiting due to network issues.

# 연결 타임아웃 5초, 읽기 타임아웃 10초
try:
    response = requests.get('https://api.example.com', timeout=(5, 10))
except requests.exceptions.Timeout:
    print("요청 시간이 초과되었습니다.")

# 전체 타임아웃 10초
response = requests.get('https://api.example.com', timeout=10)

# 타임아웃 없음 (권장하지 않음)
response = requests.get('https://api.example.com', timeout=None)

Error Handling

Different HTTP errors should be handled appropriately.

import requests
from requests.exceptions import (
    ConnectionError,
    Timeout,
    HTTPError,
    RequestException
)

try:
    response = requests.get('https://api.example.com/data', timeout=5)
    response.raise_for_status()  # 4xx, 5xx 에러 발생

    data = response.json()
    print(data)

except ConnectionError:
    print("연결 실패: 네트워크를 확인하세요.")
except Timeout:
    print("시간 초과: 서버가 응답하지 않습니다.")
except HTTPError as e:
    print(f"HTTP 에러: {e.response.status_code}")
    if e.response.status_code == 404:
        print("리소스를 찾을 수 없습니다.")
    elif e.response.status_code == 401:
        print("인증이 필요합니다.")
    elif e.response.status_code == 500:
        print("서버 내부 오류입니다.")
except RequestException as e:
    print(f"요청 실패: {str(e)}")

Using Sessions

When making multiple requests, using sessions is more efficient.

# 세션 생성
session = requests.Session()

# 모든 요청에 적용될 헤더 설정
session.headers.update({
    'User-Agent': 'MyApp/1.0',
    'Authorization': 'Bearer YOUR_TOKEN'
})

# 세션으로 여러 요청
response1 = session.get('https://api.example.com/users')
response2 = session.get('https://api.example.com/posts')
response3 = session.post('https://api.example.com/comments', json={'text': 'Hello'})

# 쿠키 자동 관리
login_data = {'username': 'john', 'password': 'secret'}
session.post('https://example.com/login', data=login_data)

# 로그인 후 인증된 요청
response = session.get('https://example.com/dashboard')

# 세션 종료
session.close()

# Context Manager 사용 (권장)
with requests.Session() as session:
    session.headers.update({'Authorization': 'Bearer TOKEN'})
    response = session.get('https://api.example.com/data')

Retry Settings

Automatic retries can be performed in case of network issues.

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

# 재시도 전략 설정
retry_strategy = Retry(
    total=3,                    # 최대 3번 재시도
    backoff_factor=1,           # 1초, 2초, 4초 대기
    status_forcelist=[429, 500, 502, 503, 504],
    method_whitelist=["HEAD", "GET", "OPTIONS", "POST"]
)

adapter = HTTPAdapter(max_retries=retry_strategy)
session = requests.Session()
session.mount("https://", adapter)
session.mount("http://", adapter)

# 자동 재시도 적용
response = session.get('https://api.example.com/data')

Practical Examples

Example 1: Getting Weather Information with OpenWeather API

import requests

def get_weather(city, api_key):
    """특정 도시의 날씨 정보를 가져옵니다."""
    base_url = "http://api.openweathermap.org/data/2.5/weather"

    params = {
        'q': city,
        'appid': api_key,
        'units': 'metric',  # 섭씨 온도
        'lang': 'kr'        # 한국어
    }

    try:
        response = requests.get(base_url, params=params, timeout=10)
        response.raise_for_status()

        data = response.json()

        weather_info = {
            'city': data['name'],
            'temperature': data['main']['temp'],
            'feels_like': data['main']['feels_like'],
            'humidity': data['main']['humidity'],
            'description': data['weather'][0]['description'],
            'wind_speed': data['wind']['speed']
        }

        return weather_info

    except requests.exceptions.RequestException as e:
        print(f"날씨 정보를 가져오는데 실패했습니다: {e}")
        return None

# 사용 예시
api_key = "YOUR_API_KEY"
weather = get_weather("Seoul", api_key)

if weather:
    print(f"도시: {weather['city']}")
    print(f"온도: {weather['temperature']}°C")
    print(f"체감온도: {weather['feels_like']}°C")
    print(f"습도: {weather['humidity']}%")
    print(f"날씨: {weather['description']}")
    print(f"풍속: {weather['wind_speed']} m/s")

Example 2: Retrieving Repository Information with GitHub API

import requests
from typing import List, Dict

class GitHubAPI:
    def __init__(self, token=None):
        self.base_url = "https://api.github.com"
        self.session = requests.Session()

        if token:
            self.session.headers.update({
                'Authorization': f'token {token}',
                'Accept': 'application/vnd.github.v3+json'
            })

    def get_user_info(self, username: str) -> Dict:
        """사용자 정보를 가져옵니다."""
        url = f"{self.base_url}/users/{username}"
        response = self.session.get(url)
        response.raise_for_status()
        return response.json()

    def get_user_repos(self, username: str) -> List[Dict]:
        """사용자의 저장소 목록을 가져옵니다."""
        url = f"{self.base_url}/users/{username}/repos"
        params = {
            'sort': 'updated',
            'per_page': 10
        }
        response = self.session.get(url, params=params)
        response.raise_for_status()
        return response.json()

    def search_repositories(self, query: str, language=None) -> List[Dict]:
        """저장소를 검색합니다."""
        url = f"{self.base_url}/search/repositories"

        search_query = query
        if language:
            search_query += f" language:{language}"

        params = {
            'q': search_query,
            'sort': 'stars',
            'order': 'desc',
            'per_page': 10
        }

        response = self.session.get(url, params=params)
        response.raise_for_status()
        return response.json()['items']

# 사용 예시
github = GitHubAPI()

# 사용자 정보 조회
user = github.get_user_info('torvalds')
print(f"이름: {user['name']}")
print(f"팔로워: {user['followers']}")
print(f"공개 저장소: {user['public_repos']}")

# 저장소 검색
repos = github.search_repositories('machine learning', language='python')
for repo in repos[:5]:
    print(f"\n{repo['full_name']}")
    print(f"⭐ {repo['stargazers_count']} | 🍴 {repo['forks_count']}")
    print(f"설명: {repo['description']}")

Example 3: Creating a REST API Client

import requests
from typing import Optional, Dict, Any
import json

class APIClient:
    """범용 REST API 클라이언트"""

    def __init__(self, base_url: str, api_key: Optional[str] = None):
        self.base_url = base_url.rstrip('/')
        self.session = requests.Session()

        # 기본 헤더 설정
        self.session.headers.update({
            'Content-Type': 'application/json',
            'User-Agent': 'Python-API-Client/1.0'
        })

        # API 키가 있으면 헤더에 추가
        if api_key:
            self.session.headers.update({'Authorization': f'Bearer {api_key}'})

    def _make_request(
        self,
        method: str,
        endpoint: str,
        **kwargs
    ) -> requests.Response:
        """HTTP 요청을 보냅니다."""
        url = f"{self.base_url}/{endpoint.lstrip('/')}"

        try:
            response = self.session.request(method, url, timeout=30, **kwargs)
            response.raise_for_status()
            return response
        except requests.exceptions.HTTPError as e:
            print(f"HTTP Error: {e.response.status_code}")
            print(f"Response: {e.response.text}")
            raise
        except requests.exceptions.RequestException as e:
            print(f"Request Error: {str(e)}")
            raise

    def get(self, endpoint: str, params: Optional[Dict] = None) -> Dict[str, Any]:
        """GET 요청"""
        response = self._make_request('GET', endpoint, params=params)
        return response.json()

    def post(self, endpoint: str, data: Optional[Dict] = None) -> Dict[str, Any]:
        """POST 요청"""
        response = self._make_request('POST', endpoint, json=data)
        return response.json()

    def put(self, endpoint: str, data: Optional[Dict] = None) -> Dict[str, Any]:
        """PUT 요청"""
        response = self._make_request('PUT', endpoint, json=data)
        return response.json()

    def patch(self, endpoint: str, data: Optional[Dict] = None) -> Dict[str, Any]:
        """PATCH 요청"""
        response = self._make_request('PATCH', endpoint, json=data)
        return response.json()

    def delete(self, endpoint: str) -> bool:
        """DELETE 요청"""
        response = self._make_request('DELETE', endpoint)
        return response.status_code in [200, 204]

# 사용 예시
client = APIClient('https://jsonplaceholder.typicode.com')

# GET 요청
posts = client.get('/posts', params={'userId': 1})
print(f"게시글 수: {len(posts)}")

# POST 요청
new_post = client.post('/posts', data={
    'title': 'New Post',
    'body': 'This is a new post',
    'userId': 1
})
print(f"생성된 게시글 ID: {new_post['id']}")

# PUT 요청
updated_post = client.put('/posts/1', data={
    'id': 1,
    'title': 'Updated Title',
    'body': 'Updated body',
    'userId': 1
})

# DELETE 요청
success = client.delete('/posts/1')
print(f"삭제 성공: {success}")

Example 4: Displaying Download Progress

import requests
from tqdm import tqdm

def download_file(url: str, filename: str):
    """파일을 다운로드하고 진행률을 표시합니다."""
    response = requests.get(url, stream=True)
    response.raise_for_status()

    total_size = int(response.headers.get('content-length', 0))

    with open(filename, 'wb') as file, tqdm(
        desc=filename,
        total=total_size,
        unit='B',
        unit_scale=True,
        unit_divisor=1024,
    ) as progress_bar:
        for chunk in response.iter_content(chunk_size=8192):
            file.write(chunk)
            progress_bar.update(len(chunk))

    print(f"다운로드 완료: {filename}")

# 사용 예시
url = "https://example.com/large-file.zip"
download_file(url, "downloaded-file.zip")

Frequently Asked Questions

Q1. What's the difference between requests and urllib?

A: requests is a more user-friendly library built on top of urllib. requests provides convenient features such as simple API, automatic JSON processing, and session management.

# urllib (복잡함)
import urllib.request
import json

request = urllib.request.Request(
    'https://api.github.com',
    headers={'User-Agent': 'Python'}
)
response = urllib.request.urlopen(request)
data = json.loads(response.read().decode())

# requests (간단함)
import requests

response = requests.get('https://api.github.com')
data = response.json()

Q2. How do you download large files?

A: Use the stream=True option to download in chunks.

response = requests.get(url, stream=True)
with open('large_file.zip', 'wb') as f:
    for chunk in response.iter_content(chunk_size=8192):
        if chunk:
            f.write(chunk)

Q3. Getting an SSL certificate error.

A: In development environments, you can use verify=False, but this is not recommended in production.

# 개발 환경에서만 사용
response = requests.get(url, verify=False)

# 프로덕션: 인증서 경로 지정
response = requests.get(url, verify='/path/to/certfile')

Q4. How do you configure a proxy?

A: Use the proxies parameter.

proxies = {
    'http': 'http://10.10.10.10:8000',
    'https': 'http://10.10.10.10:8000',
}

response = requests.get('https://api.example.com', proxies=proxies)

Q5. What if the response isn't JSON?

A: Use the appropriate attribute depending on the response type.

response = requests.get(url)

# JSON
data = response.json()

# 텍스트
text = response.text

# 바이너리
binary = response.content

# 자동 감지
if 'application/json' in response.headers.get('Content-Type', ''):
    data = response.json()
else:
    text = response.text

Next Steps

If you've mastered HTTP requests with requests, learn these topics:

1. Web Scraping - Extract data from web pages with BeautifulSoup
2. Asynchronous Requests - Process multiple requests simultaneously with aiohttp
3. API Development - Create your own REST API with FastAPI
4. Authentication - Advanced authentication methods like OAuth 2.0, JWT

References

• Requests Official Documentation
• HTTP Status Codes
• REST API Design Guide