File I/O & Networking Pro Tips

io.Reader Pipeline Composition Pattern

Combining multiple io.Readers lets you process compression, encryption, and hash calculation all at the same time.

package main

import (
	"compress/gzip"
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"strings"
)

// read a file while simultaneously computing SHA-256 and gzip compressing
func compressWithHash(src io.Reader, dst io.Writer) (hashHex string, bytesRead int64, err error) {
	hasher := sha256.New()

	// TeeReader: read from src while feeding the same data to hasher
	tee := io.TeeReader(src, hasher)

	gz := gzip.NewWriter(dst)
	defer gz.Close()

	bytesRead, err = io.Copy(gz, tee)
	if err != nil {
		return "", 0, err
	}

	return fmt.Sprintf("%x", hasher.Sum(nil)), bytesRead, nil
}

func main() {
	src := strings.NewReader("long text data to compress and hash...")
	var compressed strings.Builder

	hash, n, err := compressWithHash(src, &compressed)
	if err != nil {
		fmt.Println("failed:", err)
		return
	}

	fmt.Printf("original size: %d bytes\n", n)
	fmt.Printf("compressed size: %d bytes\n", compressed.Len())
	fmt.Printf("SHA-256: %s\n", hash)
	_ = os.Stderr // suppress compiler warning
}

Large File Processing — Reading in Chunks

package main

import (
	"fmt"
	"io"
	"os"
)

// processChunks: process a large file in fixed-size chunks
func processChunks(filename string, chunkSize int, process func(chunk []byte, chunkNum int) error) error {
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	buf := make([]byte, chunkSize) // reuse buffer — allocated only once
	chunkNum := 0

	for {
		n, err := io.ReadFull(f, buf)
		if n > 0 {
			chunkNum++
			if procErr := process(buf[:n], chunkNum); procErr != nil {
				return procErr
			}
		}
		if err == io.EOF || err == io.ErrUnexpectedEOF {
			break // end of file
		}
		if err != nil {
			return err
		}
	}

	fmt.Printf("processed %d chunks total\n", chunkNum)
	return nil
}

func main() {
	// create test file (1MB)
	data := make([]byte, 1024*1024)
	for i := range data {
		data[i] = byte(i % 256)
	}
	os.WriteFile("large.bin", data, 0644)
	defer os.Remove("large.bin")

	// process in 64KB chunks
	err := processChunks("large.bin", 64*1024, func(chunk []byte, n int) error {
		fmt.Printf("chunk %d: %d bytes\n", n, len(chunk))
		return nil
	})
	if err != nil {
		fmt.Println("processing failed:", err)
	}
}

bufio.Scanner Token Size Limit and Solution

The default bufio.Scanner buffer is 64KB. If a single line exceeds 64KB, you'll get a bufio.Scanner: token too long error.

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	// generate a very long line (128KB)
	longLine := strings.Repeat("A", 128*1024)
	text := "short line\n" + longLine + "\nanother line\n"

	// default Scanner — will error
	scanner1 := bufio.NewScanner(strings.NewReader(text))
	for scanner1.Scan() {
		// stops on the long line
	}
	if err := scanner1.Err(); err != nil {
		fmt.Println("default scanner error:", err) // "token too long"
	}

	// solution 1: expand buffer with Scanner.Buffer
	scanner2 := bufio.NewScanner(strings.NewReader(text))
	scanner2.Buffer(make([]byte, 256*1024), 256*1024) // 256KB buffer
	for scanner2.Scan() {
		fmt.Printf("line length: %d\n", len(scanner2.Text()))
	}
	fmt.Println("scanner2 error:", scanner2.Err()) // nil

	// solution 2: use bufio.NewReader.ReadString (no size limit)
	reader := bufio.NewReaderSize(strings.NewReader(text), 64*1024)
	for {
		line, err := reader.ReadString('\n')
		if len(line) > 0 {
			fmt.Printf("ReadString line length: %d\n", len(line))
		}
		if err != nil {
			break
		}
	}
}

Network Timeouts — Separating connect·read·write

package main

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"time"
)

func main() {
	// three timeout stages, configured separately:
	// 1. TCP connection timeout (DialContext)
	// 2. TLS handshake timeout (TLSHandshakeTimeout)
	// 3. response header timeout (ResponseHeaderTimeout)
	// 4. total request timeout (http.Client.Timeout)
	transport := &http.Transport{
		DialContext: (&net.Dialer{
			Timeout:   3 * time.Second, // TCP connection timeout only
			KeepAlive: 30 * time.Second,
		}).DialContext,
		TLSHandshakeTimeout:   5 * time.Second,  // TLS negotiation timeout
		ResponseHeaderTimeout: 10 * time.Second, // header receive timeout
	}

	client := &http.Client{
		Transport: transport,
		Timeout:   30 * time.Second, // total timeout (final safety net)
	}

	// cancelable request via context
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	req, _ := http.NewRequestWithContext(ctx, "GET", "https://example.com", nil)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("request failed:", err)
		return
	}
	defer resp.Body.Close()
	fmt.Println("status:", resp.StatusCode)
}

HTTP Client Connection Pool Reuse

package main

import (
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"
)

func main() {
	// wrong: creating http.Client per request — no connection pool benefit
	// for i := 0; i < 100; i++ {
	//     client := &http.Client{}
	//     resp, _ := client.Get("https://example.com")
	//     resp.Body.Close()
	// }

	// correct: share one http.Client (goroutine-safe)
	client := &http.Client{
		Transport: &http.Transport{
			MaxIdleConnsPerHost: 20, // keep up to 20 idle connections per host
		},
		Timeout: 10 * time.Second,
	}

	var wg sync.WaitGroup
	start := time.Now()

	// 10 concurrent requests — fast with connection pool reuse
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func(n int) {
			defer wg.Done()
			resp, err := client.Get("https://httpbin.org/get")
			if err != nil {
				fmt.Printf("request %d failed: %v\n", n, err)
				return
			}
			defer resp.Body.Close()
			// must read Body completely for connection to be reused!
			io.Copy(io.Discard, resp.Body)
			fmt.Printf("request %d done: %d\n", n, resp.StatusCode)
		}(i)
	}

	wg.Wait()
	fmt.Printf("total elapsed: %v\n", time.Since(start))
}

Key point: If you close the resp.Body without reading it, that TCP connection won't be returned to the pool. Call io.Copy(io.Discard, resp.Body) followed by resp.Body.Close() for connections to be reused.

Preventing File Descriptor Leaks

package main

import (
	"fmt"
	"os"
)

// bad pattern: possible fd leak in error branches
func badPattern(path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	// if an error occurs in the middle, f.Close() may be missed
	data := make([]byte, 100)
	n, err := f.Read(data)
	if err != nil {
		return err // f is not closed!
	}
	fmt.Printf("bytes read: %d\n", n)
	f.Close()
	return nil
}

// good pattern: defer guarantees close on all return paths
func goodPattern(path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close() // runs automatically on every return path

	data := make([]byte, 100)
	n, err := f.Read(data)
	if err != nil {
		return err // defer ensures f.Close() is called
	}
	fmt.Printf("bytes read: %d\n", n)
	return nil
}

// handling multiple files — don't defer inside a loop, close immediately
func processFiles(paths []string) error {
	for _, path := range paths {
		// bad: all files stay open until function returns
		// f, _ := os.Open(path)
		// defer f.Close()

		// good: close via closure when done with each file
		err := func(p string) error {
			f, err := os.Open(p)
			if err != nil {
				return err
			}
			defer f.Close() // closed when this closure returns
			fmt.Println("processing:", p)
			return nil
		}(path)
		if err != nil {
			return err
		}
	}
	return nil
}

func main() {
	// create test file
	os.WriteFile("test.txt", []byte("test"), 0644)
	defer os.Remove("test.txt")

	goodPattern("test.txt")
	processFiles([]string{"test.txt"})
}

Writing Temp Files Safely — Atomic Write Pattern

package main

import (
	"fmt"
	"os"
	"path/filepath"
)

// atomicWriteFile: write to a temp file then atomically replace the target
// (preserves original on failure)
func atomicWriteFile(path string, data []byte, perm os.FileMode) error {
	// create temp file in same directory (cross-partition Rename would fail)
	dir := filepath.Dir(path)
	tmp, err := os.CreateTemp(dir, ".tmp-*")
	if err != nil {
		return fmt.Errorf("temp file creation failed: %w", err)
	}
	tmpPath := tmp.Name()

	// cleanup temp file on failure
	success := false
	defer func() {
		if !success {
			os.Remove(tmpPath)
		}
	}()

	// write to temp file
	if _, err := tmp.Write(data); err != nil {
		tmp.Close()
		return fmt.Errorf("temp file write failed: %w", err)
	}

	// flush to disk (data safety)
	if err := tmp.Sync(); err != nil {
		tmp.Close()
		return fmt.Errorf("sync failed: %w", err)
	}

	if err := tmp.Close(); err != nil {
		return fmt.Errorf("temp file close failed: %w", err)
	}

	// set permissions
	if err := os.Chmod(tmpPath, perm); err != nil {
		return fmt.Errorf("chmod failed: %w", err)
	}

	// atomic replacement: original is safe even if this step fails
	if err := os.Rename(tmpPath, path); err != nil {
		return fmt.Errorf("file replace failed: %w", err)
	}

	success = true
	return nil
}

func main() {
	content := []byte("important configuration data\nversion: 2.0\n")

	if err := atomicWriteFile("config.json", content, 0644); err != nil {
		fmt.Println("atomic write failed:", err)
		return
	}
	fmt.Println("config file saved safely")

	// verify
	data, _ := os.ReadFile("config.json")
	fmt.Printf("saved content:\n%s", data)

	os.Remove("config.json")
}

Why the Atomic Write Pattern matters:

• If a process dies while writing directly to a file, the file is left in a partially written state
• Writing completely to a temp file then using os.Rename guarantees an atomic operation (within the same partition)
• Always apply this to critical files like config files, database files, and cache files