File I/O — Mastering the os·bufio·io Packages

File reading and writing are at the heart of every practical program. Go provides powerful and flexible file I/O through the combination of three packages: os, bufio, and io. Following Unix philosophy, files are treated as streams, allowing files, network connections, and memory to be handled transparently through the same interface.

os Package — Opening, Creating, and Deleting Files

Opening and Creating Files

package main

import (
	"fmt"
	"os"
)

func main() {
	// Open for reading only
	f, err := os.Open("data.txt")
	if err != nil {
		fmt.Println("open failed:", err)
		return
	}
	defer f.Close() // must close to prevent fd leaks

	// Create file (creates new, truncates if exists)
	fw, err := os.Create("output.txt")
	if err != nil {
		fmt.Println("create failed:", err)
		return
	}
	defer fw.Close()
	fw.WriteString("Hello, Go!\n")

	// OpenFile for fine-grained control
	// os.O_APPEND: append to end, os.O_CREATE: create if missing, os.O_WRONLY: write only
	fa, err := os.OpenFile("log.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		fmt.Println("OpenFile failed:", err)
		return
	}
	defer fa.Close()
	fa.WriteString("log message\n")

	fmt.Println("file operations complete")
}

Understanding File Permissions (FileMode):

Value	Meaning
0644	Owner read/write, group/others read
0755	Owner full access, group/others read/execute
0600	Owner read/write only (for sensitive files)

Reading and Writing Entire Files (Go 1.16+)

package main

import (
	"fmt"
	"os"
)

func main() {
	// Read entire file at once (suitable for small files)
	data, err := os.ReadFile("data.txt")
	if err != nil {
		fmt.Println("read failed:", err)
		return
	}
	fmt.Printf("read %d bytes:\n%s", len(data), data)

	// Write entire file at once
	content := []byte("new content\nsecond line\n")
	err = os.WriteFile("output.txt", content, 0644)
	if err != nil {
		fmt.Println("write failed:", err)
		return
	}
	fmt.Println("file write successful")
}

File Info, Deletion, and Renaming

package main

import (
	"fmt"
	"os"
)

func main() {
	// Query file info
	info, err := os.Stat("data.txt")
	if err != nil {
		if os.IsNotExist(err) {
			fmt.Println("file does not exist")
		} else {
			fmt.Println("stat error:", err)
		}
		return
	}
	fmt.Printf("name: %s\n", info.Name())
	fmt.Printf("size: %d bytes\n", info.Size())
	fmt.Printf("modified: %v\n", info.ModTime())
	fmt.Printf("is directory: %v\n", info.IsDir())
	fmt.Printf("permissions: %v\n", info.Mode())

	// Create directory
	err = os.Mkdir("mydir", 0755)
	if err != nil && !os.IsExist(err) {
		fmt.Println("mkdir failed:", err)
	}

	// Create nested directories (like mkdir -p)
	err = os.MkdirAll("a/b/c", 0755)
	if err != nil {
		fmt.Println("MkdirAll failed:", err)
	}

	// Rename / move file
	err = os.Rename("old.txt", "new.txt")
	if err != nil {
		fmt.Println("rename failed:", err)
	}

	// Remove file
	err = os.Remove("temp.txt")
	if err != nil && !os.IsNotExist(err) {
		fmt.Println("remove failed:", err)
	}
}

io Package — The Heart of Stream Interfaces

Go's I/O philosophy starts with the io.Reader and io.Writer interfaces. Understanding these two interfaces lets you work with files, networks, compression, and encryption all in the same way.

Core Interfaces

// Core interfaces defined in the io package
type Reader interface {
    Read(p []byte) (n int, err error)
}

type Writer interface {
    Write(p []byte) (n int, err error)
}

type Closer interface {
    Close() error
}

// Composite interfaces
type ReadWriter interface {
    Reader
    Writer
}

type ReadCloser interface {
    Reader
    Closer
}

io.Copy — The Core of Stream Copying

package main

import (
	"fmt"
	"io"
	"os"
	"strings"
)

func main() {
	// io.Copy: copy everything from src to dst until EOF
	src := strings.NewReader("content to copy — handles large text efficiently")
	dst, err := os.Create("copied.txt")
	if err != nil {
		fmt.Println("file create failed:", err)
		return
	}
	defer dst.Close()

	n, err := io.Copy(dst, src)
	if err != nil {
		fmt.Println("copy failed:", err)
		return
	}
	fmt.Printf("%d bytes copied\n", n)

	// io.ReadAll: read entire Reader into []byte
	r := strings.NewReader("full content")
	data, err := io.ReadAll(r)
	if err != nil {
		fmt.Println("ReadAll failed:", err)
		return
	}
	fmt.Println("read content:", string(data))

	// io.LimitReader: limit maximum bytes to read
	limited := io.LimitReader(strings.NewReader("abcdefghij"), 5)
	data, _ = io.ReadAll(limited)
	fmt.Println("limited read:", string(data)) // "abcde"
}

io.MultiReader·MultiWriter·TeeReader

package main

import (
	"fmt"
	"io"
	"os"
	"strings"
)

func main() {
	// MultiReader: chain multiple Readers sequentially
	r1 := strings.NewReader("first part ")
	r2 := strings.NewReader("second part ")
	r3 := strings.NewReader("third part")
	multi := io.MultiReader(r1, r2, r3)

	data, _ := io.ReadAll(multi)
	fmt.Println("combined:", string(data))

	// MultiWriter: write once to multiple Writers simultaneously
	f, _ := os.Create("multiwrite.txt")
	defer f.Close()

	// Write to both file and stdout at the same time
	mw := io.MultiWriter(f, os.Stdout)
	fmt.Fprintln(mw, "written to both file and console!")

	// TeeReader: read and simultaneously write to another Writer (stream tapping)
	src := strings.NewReader("original data stream")
	var buf strings.Builder
	tee := io.TeeReader(src, &buf) // reads from src while copying to buf

	// reading from tee also records to buf
	data, _ = io.ReadAll(tee)
	fmt.Println("tee read:", string(data))
	fmt.Println("buffer content:", buf.String())
}

io.Pipe — Synchronous Pipe

package main

import (
	"fmt"
	"io"
)

func main() {
	// io.Pipe: synchronous in-memory pipe (useful for passing streams between goroutines)
	pr, pw := io.Pipe()

	// writer goroutine
	go func() {
		defer pw.Close()
		for i := 1; i <= 5; i++ {
			fmt.Fprintf(pw, "message %d\n", i)
		}
	}()

	// reader (main goroutine)
	data, err := io.ReadAll(pr)
	if err != nil {
		fmt.Println("pipe read failed:", err)
		return
	}
	fmt.Print(string(data))
}

bufio Package — Buffered I/O

System calls are expensive. bufio maintains an internal buffer to reduce system call frequency and greatly improve performance.

bufio.NewReader and bufio.NewWriter

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
)

func main() {
	// bufio.NewReader: buffered reading
	r := bufio.NewReader(strings.NewReader("first line\nsecond line\nthird line\n"))

	// read line by line
	for {
		line, err := r.ReadString('\n') // read up to delimiter
		if len(line) > 0 {
			fmt.Print("read line:", line)
		}
		if err != nil {
			break // includes io.EOF
		}
	}

	// bufio.NewWriter: buffered writing (must call Flush!)
	f, _ := os.Create("buffered.txt")
	defer f.Close()

	w := bufio.NewWriter(f)
	for i := 1; i <= 5; i++ {
		fmt.Fprintf(w, "line %d\n", i)
	}
	// Without Flush, buffered content won't be written to file!
	if err := w.Flush(); err != nil {
		fmt.Println("flush failed:", err)
	}
	fmt.Println("buffered write complete")
}

bufio.Scanner — Flexible Token Scanning

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	text := "first line\nsecond line\nthird line"

	// ScanLines (default): scan line by line
	scanner := bufio.NewScanner(strings.NewReader(text))
	lineNum := 0
	for scanner.Scan() {
		lineNum++
		fmt.Printf("line %d: %s\n", lineNum, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		fmt.Println("scan error:", err)
	}

	// ScanWords: scan word by word
	wordText := "Go language is fast and concise"
	scanner2 := bufio.NewScanner(strings.NewReader(wordText))
	scanner2.Split(bufio.ScanWords)
	for scanner2.Scan() {
		fmt.Printf("word: [%s]\n", scanner2.Text())
	}

	// ScanBytes: scan byte by byte
	byteText := "ABC"
	scanner3 := bufio.NewScanner(strings.NewReader(byteText))
	scanner3.Split(bufio.ScanBytes)
	for scanner3.Scan() {
		fmt.Printf("byte: %v\n", scanner3.Bytes())
	}
}

File Seeking (Seek)

package main

import (
	"fmt"
	"io"
	"os"
)

func main() {
	// Create test file
	os.WriteFile("seek_test.txt", []byte("0123456789ABCDEF"), 0644)

	f, _ := os.Open("seek_test.txt")
	defer f.Close()

	buf := make([]byte, 4)

	// Read first 4 bytes from start
	f.Read(buf)
	fmt.Println("first 4 bytes:", string(buf)) // "0123"

	// Absolute seek: move to byte 8 from start (SeekStart)
	pos, _ := f.Seek(8, io.SeekStart)
	fmt.Println("position after seek:", pos)
	f.Read(buf)
	fmt.Println("4 bytes from position 8:", string(buf)) // "89AB"

	// Relative seek: move back 2 bytes from current
	f.Seek(-2, io.SeekCurrent)
	f.Read(buf)
	fmt.Println("4 bytes after -2 from current:", string(buf)) // "ABCD"

	// Seek from end: move -4 bytes from end (SeekEnd)
	f.Seek(-4, io.SeekEnd)
	f.Read(buf)
	fmt.Println("4 bytes from -4 at end:", string(buf)) // "CDEF"

	// Check current position (0-move trick)
	currentPos, _ := f.Seek(0, io.SeekCurrent)
	fmt.Println("current file pointer position:", currentPos)
}

Real-World Example 1 — Log File Analyzer

An analyzer that extracts only ERROR-level entries from large log files.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
	"time"
)

type LogEntry struct {
	Level   string
	Message string
	Line    int
}

// analyzeLog: extract entries of a specific level from a log file
func analyzeLog(filename string, targetLevel string) ([]LogEntry, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("failed to open file: %w", err)
	}
	defer f.Close()

	var entries []LogEntry
	scanner := bufio.NewScanner(f)
	lineNum := 0

	for scanner.Scan() {
		lineNum++
		line := scanner.Text()

		// format: "[LEVEL] message"
		if strings.HasPrefix(line, "["+targetLevel+"]") {
			msg := strings.TrimPrefix(line, "["+targetLevel+"] ")
			entries = append(entries, LogEntry{
				Level:   targetLevel,
				Message: msg,
				Line:    lineNum,
			})
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("scan error: %w", err)
	}

	return entries, nil
}

// writeReport: save analysis results to a file
func writeReport(filename string, entries []LogEntry) error {
	f, err := os.Create(filename)
	if err != nil {
		return fmt.Errorf("failed to create report file: %w", err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	fmt.Fprintf(w, "Log Analysis Report — %s\n", time.Now().Format("2006-01-02 15:04:05"))
	fmt.Fprintf(w, "Total errors found: %d\n\n", len(entries))

	for _, entry := range entries {
		fmt.Fprintf(w, "line %4d | %s\n", entry.Line, entry.Message)
	}

	return w.Flush()
}

func main() {
	// Create sample log file
	sampleLog := `[INFO] server started
[INFO] database connection successful
[ERROR] query execution failed: connection timeout
[INFO] processing request
[ERROR] authentication failed: invalid token
[WARN] memory usage exceeds 80%
[ERROR] file not found: config.yaml
[INFO] server shutting down gracefully
`
	os.WriteFile("app.log", []byte(sampleLog), 0644)

	// analyze error logs
	errors, err := analyzeLog("app.log", "ERROR")
	if err != nil {
		fmt.Println("analysis failed:", err)
		return
	}

	fmt.Printf("found %d ERRORs\n", len(errors))
	for _, e := range errors {
		fmt.Printf("  line %d: %s\n", e.Line, e.Message)
	}

	// save report
	if err := writeReport("error_report.txt", errors); err != nil {
		fmt.Println("failed to save report:", err)
		return
	}
	fmt.Println("report saved to error_report.txt")

	// cleanup
	os.Remove("app.log")
	os.Remove("error_report.txt")
}

Real-World Example 2 — Large CSV Streaming

Processing multi-GB CSV files while keeping memory usage low.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

type SalesRecord struct {
	Region  string
	Product string
	Amount  float64
}

// processCSVStream: process large CSV files via streaming
// processes one line at a time without loading the entire file into memory
func processCSVStream(filename string) (map[string]float64, error) {
	f, err := os.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("failed to open CSV: %w", err)
	}
	defer f.Close()

	// use a larger buffer than the default (4096) for better performance
	scanner := bufio.NewScanner(f)
	scanner.Buffer(make([]byte, 1024*1024), 1024*1024) // 1MB buffer

	// total sales by region
	regionSales := make(map[string]float64)
	lineNum := 0

	for scanner.Scan() {
		lineNum++
		line := scanner.Text()

		// skip header line
		if lineNum == 1 {
			continue
		}

		// CSV parsing (simple example — use encoding/csv in production)
		fields := strings.Split(line, ",")
		if len(fields) < 3 {
			fmt.Printf("line %d: insufficient fields, skipping\n", lineNum)
			continue
		}

		amount, err := strconv.ParseFloat(strings.TrimSpace(fields[2]), 64)
		if err != nil {
			fmt.Printf("line %d: amount parse failed (%s), skipping\n", lineNum, fields[2])
			continue
		}

		region := strings.TrimSpace(fields[0])
		regionSales[region] += amount
	}

	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("CSV scan error at line %d: %w", lineNum, err)
	}

	fmt.Printf("processed %d lines total (%d records)\n", lineNum, lineNum-1)
	return regionSales, nil
}

func main() {
	// Create sample CSV file
	csvData := "region,product,amount\nSeoul,Laptop,1200000\nBusan,Mouse,35000\nSeoul,Keyboard,89000\nDaegu,Monitor,450000\nBusan,Laptop,1150000\nSeoul,Headset,120000\n"
	os.WriteFile("sales.csv", []byte(csvData), 0644)

	// streaming processing
	sales, err := processCSVStream("sales.csv")
	if err != nil {
		fmt.Println("processing failed:", err)
		return
	}

	fmt.Println("\nSales by region:")
	total := 0.0
	for region, amount := range sales {
		fmt.Printf("  %-10s: %12.0f\n", region, amount)
		total += amount
	}
	fmt.Printf("  %-10s: %12.0f\n", "Total", total)

	os.Remove("sales.csv")
}

Pro Tips

Always defer file close immediately: Get into the habit of writing defer f.Close() right after os.Open. It will never be missed no matter how many error branches you have.

bufio.Writer Flush is mandatory: If you forget to call Flush() after using bufio.NewWriter, the buffered data won't be written to the file. Use defer w.Flush() for safety.

Small files: os.ReadFile, large files: Scanner: For files under a few MB, os.ReadFile is simple and sufficient. For files over tens of MB, use bufio.Scanner for line-by-line streaming.

io.Copy is more efficient than os.ReadFile when forwarding: When reading a file and passing it to another Writer (network, compression stream, etc.), io.Copy saves memory by avoiding an intermediate buffer.