Developer Environment Setup Runbook

wsl --install -d Ubuntu

Ubuntu has been installed. Restart your machine to finish setup.

# 1. Print Working Directory: see where you currently stand
pwd

# 2. List files: -l shows permissions and sizes, -a reveals hidden config files like .env
ls -la

# 3. Create a project directory tree safely with parent folders (-p)
mkdir -p ~/projects/shopkart-pipeline/src

# 4. Change directory into your new project folder
cd ~/projects/shopkart-pipeline

# 5. Move back up one folder level
cd ..

# Export variable in current shell session
export DB_PORT="5432"
export PIPELINE_ENV="development"

# Verify variable value
echo "Connecting on port: $DB_PORT"

# Run a Python one-liner to verify Python reads the variable
python3 -c "import os; print('Python saw port:', os.getenv('DB_PORT'))"

Connecting on port: 5432\nPython saw port: 5432

python3 --version
pip3 --version

Python 3.11.8 (or higher)\npip 24.0 from ...

# 1. Navigate to your project folder
cd ~/projects/shopkart-pipeline

# 2. Create the virtual environment folder
python3 -m venv .venv

# 3. Activate the environment (macOS / Linux / WSL)
source .venv/bin/activate

# 4. Verify your active python binary points inside .venv
which python

/Users/yourname/projects/shopkart-pipeline/.venv/bin/python

# If PowerShell restricts scripts, enable local execution once:
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser

# Activate the environment:
.\.venv\Scripts\Activate.ps1

# 1. Always upgrade pip inside a fresh environment
pip install --upgrade pip

# 2. Install core libraries needed for our curriculum
pip install duckdb pandas pytest pydantic requests

# 3. Export exact pinned versions to requirements.txt
pip freeze > requirements.txt

# 4. Review the generated requirements
head -n 5 requirements.txt

git config --global user.name "Your Full Name"
git config --global user.email "[email protected]"
git config --global init.defaultBranch main

cat << 'EOF' > .gitignore
# Virtual environments
.venv/
env/
venv/

# Environment files and credentials (never commit secrets!)
.env
.env.local
*.pem
credentials.json

# Byte-compiled / cache files
__pycache__/
*.py[cod]
.pytest_cache/

# Local raw data and Parquet caches
data/raw/
data/bronze/
*.parquet
*.csv

# Operating system noise
.DS_Store
Thumbs.db
EOF

# Step 1: Initialize local repository (if new project)
git init

# Step 2: Create and switch to a descriptive feature branch
git checkout -b feat/ingest-orders-bronze

# Step 3: Check status and stage modified files
git status
git add src/ingest.py requirements.txt .gitignore

# Step 4: Create a concise, meaningful commit
git commit -m "feat(ingest): add keyset pagination extractor for orders"

# Step 5: Link remote and push branch to GitHub
git remote add origin https://github.com/your-username/shopkart-pipeline.git
git push -u origin feat/ingest-orders-bronze

docker --version
docker compose version
docker run --rm hello-world

Hello from Docker! This message shows that your installation appears to be working correctly.

# 1. Use a lightweight, official Python base image
FROM python:3.11-slim

# 2. Set working directory inside container
WORKDIR /app

# 3. Copy only dependency manifest first to leverage Docker layer caching
COPY requirements.txt .

# 4. Install dependencies without caching downloaded wheels to minimize image size
RUN pip install --no-cache-dir -r requirements.txt

# 5. Copy pipeline source code
COPY src/ ./src/

# 6. Default command to execute when container starts
CMD ["python", "src/ingest.py"]

# Build the image from current directory (.)
docker build -t shopkart-ingest:v1 .

# Run container and automatically remove it (--rm) upon completion
docker run --rm -e PIPELINE_ENV=production shopkart-ingest:v1

version: "3.8"

services:
  postgres:
    image: postgres:15-alpine
    container_name: shopkart-postgres
    restart: always
    environment:
      POSTGRES_USER: shopkart_user
      POSTGRES_PASSWORD: shopkart_password
      POSTGRES_DB: shopkart_oltp
    ports:
      - "5432:5432"
    volumes:
      - pgdata:/var/lib/postgresql/data

volumes:
  pgdata:

# Start stack in background (-d = detached)
docker compose up -d

# Check running container status
docker compose ps

# Follow live output logs for the database container (-f = follow)
docker compose logs -f postgres

# Stop and remove containers when you are done working
docker compose down