PowerShell to Go Migration: Containerized Web Scraper Project

// Streamlined data processing pipeline
func main() {
    // 1. Fetch live data from external API
    livePosts, err := fetchLivePostsFromAPI()
    
    // 2. Load translation mappings
    translations, err := loadTranslationsFromCSV()
    
    // 3. Process active posts with translations
    processedData := processPostsWithTranslations(livePosts, translations)
    
    // 4. Generate multi-format outputs
    generateOutputs(processedData)
}

# Multi-stage build for optimal size
FROM golang:1.23-alpine AS builder
# ... build process ...

FROM alpine:latest
# Runtime image: ~50MB total
COPY --from=builder /app/scraper /app/scraper
USER 65534  # Non-root security

apiVersion: batch/v1
kind: CronJob
spec:
  schedule: "0 */4 * * *"  # Every 4 hours
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: scraper
            image: ghcr.io/username/scraper:latest
            imagePullPolicy: Always
            resources:
              requests:
                memory: "256Mi"
                cpu: "200m"
              limits:
                memory: "1Gi"
                cpu: "1000m"

// External API integration replacing CSV dependency
type PostData struct {
    ID              string `json:"id"`
    EnglishName     string `json:"name"`
    CountryCode     string `json:"country"`
    SchedulingURL   string `json:"url"`
    Data       []string `json:"data"`
}

// Live data fetching
func fetchLivePostsFromAPI() ([]PostData, error) {
    resp, err := http.Get("https://api.example.com/data/posts.js")
    // Parse and validate data
    return posts, nil
}

// Concurrent output generation
func generateOutputs(data []PostData) {
    var wg sync.WaitGroup
    
    // HTML generation
    wg.Add(1)
    go func() {
        defer wg.Done()
        generateHTML(data)
    }()
    
    // Multi-format images (PNG, WebP, AVIF)
    wg.Add(1)
    go func() {
        defer wg.Done()
        generateImages(data)
    }()
    
    wg.Wait()
}

// S3-compatible upload with retry logic
type Uploader struct {
    client   *s3.Client
    bucket   string
    endpoint string
}

func (u *Uploader) UploadWithRetry(file string) error {
    for attempt := 1; attempt <= maxRetries; attempt++ {
        if err := u.upload(file); err == nil {
            return nil
        }
        time.Sleep(backoffDelay * time.Duration(attempt))
    }
    return fmt.Errorf("upload failed after %d attempts", maxRetries)
}

// Align Go implementation with PowerShell data flow
func processData() []PostData {
    // 1. Fetch live posts from API (247 active posts)
    livePosts := fetchFromAPI()
    
    // 2. Load translation mappings (245 translations)  
    translations := loadTranslations()
    
    // 3. Apply translations to live data
    for i, post := range livePosts {
        if trans, exists := translations[post.ID]; exists {
            livePosts[i].LocalizedName = trans.LocalizedName
            livePosts[i].LocalizedCountry = trans.LocalizedCountry
        }
        // Use English fallback if no translation
    }
    
    return livePosts
}

# GitHub Actions workflow
name: Build and Deploy
on:
  push:
    branches: [ main ]
    
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-go@v4
      with:
        go-version: '1.23'
    
    - name: Run tests
      run: go test -v ./...
    
    - name: Build and push Docker image
      uses: docker/build-push-action@v5
      with:
        push: true
        tags: ghcr.io/username/scraper:latest

# Kubernetes GitOps deployment
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: web-scraper
spec:
  interval: 10m
  path: ./apps/scraper/base
  prune: true
  sourceRef:
    kind: GitRepository
    name: flux-system

# Development to production pipeline
git push origin main                    # 1. Push code changes
# → GitHub Actions builds new image     # 2. Automatic build
kubectl delete cronjob scraper -n app  # 3. Force deployment
# → FluxCD recreates with latest image  # 4. Automatic deployment

# Security-hardened container
RUN apk add --no-cache ca-certificates tzdata
RUN adduser -D -s /bin/sh scraper
USER scraper:scraper

# Read-only root filesystem
VOLUME ["/tmp"]
USER 65534

# Pod security context
securityContext:
  runAsNonRoot: true
  runAsUser: 65534
  readOnlyRootFilesystem: true
  allowPrivilegeEscalation: false
  capabilities:
    drop: ["ALL"]

# SOPS-encrypted secrets for sensitive data
apiVersion: v1
kind: Secret
metadata:
  name: scraper-credentials
stringData:
  api-key: ENC[AES256_GCM,data:encrypted-data,type:str]