BrevFeed
Sign in

63. Capstone 1: a word-frequency report

📖 Reading · 12 min
💡 Most code boxes below are live — edit one and hit Run. Boxes without a Run button are reference-only (they can't run in your browser).

Time to build something whole. This one is deliberately unglamorous — read text, count words, print a ranked report — because it uses almost everything from the first half of the course: strings and runes, maps, slices, structs, sorting, and clean error handling.

The specification

Given a block of text, produce a report showing:

  • total word count and unique word count
  • the top N words by frequency, ties broken alphabetically
  • words shorter than a minimum length excluded ("stop words")

Step 1: tokenising

Splitting on spaces isn't enough — real text has punctuation, mixed case and newlines.

package main

import (
    "fmt"
    "strings"
    "unicode"
)

func tokenize(text string) []string {
    fields := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
        return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\''
    })
    return fields
}

func main() {
    text := `The quick brown fox jumps over the lazy dog.
The dog barks; the fox doesn't care!`

    words := tokenize(text)
    fmt.Println(len(words), "words")
    fmt.Println(words[:6])
    fmt.Println(words[len(words)-4:])
}

strings.FieldsFunc splits wherever your function says a rune is a separator. We keep letters, digits and apostrophes (so doesn't stays one word) and treat everything else as a boundary. Lower-casing first means The and the count together.

That's unicode.IsLetter, not r >= 'a' && r <= 'z' — the difference matters the moment somebody pastes text with an accent in it.

Step 2: counting

package main

import (
    "fmt"
    "strings"
    "unicode"
)

func tokenize(text string) []string {
    return strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
        return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\''
    })
}

func count(words []string, minLen int) map[string]int {
    counts := make(map[string]int)
    for _, w := range words {
        if len(w) < minLen {
            continue
        }
        counts[w]++
    }
    return counts
}

func main() {
    text := "the quick brown fox the lazy dog the fox"

    counts := count(tokenize(text), 3)

    fmt.Println("unique words of 3+ letters:", len(counts))
    fmt.Println("the:", counts["the"], "fox:", counts["fox"])
    fmt.Println("never seen:", counts["elephant"])
}

counts[w]++ on an absent key gives 1 — module 4's payoff. And reading a word that was never counted returns 0 rather than an error, so the report needs no special case for missing words.

Step 3: ranking

A map has no order, so we build a sortable slice of structs:

package main

import (
    "fmt"
    "sort"
    "strings"
    "unicode"
)

type WordCount struct {
    Word  string
    Count int
}

func tokenize(text string) []string {
    return strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
        return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\''
    })
}

func rank(counts map[string]int) []WordCount {
    out := make([]WordCount, 0, len(counts))
    for w, n := range counts {
        out = append(out, WordCount{Word: w, Count: n})
    }
    sort.Slice(out, func(i, j int) bool {
        if out[i].Count != out[j].Count {
            return out[i].Count > out[j].Count
        }
        return out[i].Word < out[j].Word
    })
    return out
}

func main() {
    text := "go is fast go is simple go is fun rust is fast"

    counts := make(map[string]int)
    for _, w := range tokenize(text) {
        counts[w]++
    }

    for i, wc := range rank(counts) {
        fmt.Printf("%d. %-6s %d\n", i+1, wc.Word, wc.Count)
    }
}

The comparator is the important line: count descending, then word ascending. Without that second clause, words with equal counts would come out in the map's random order and the report would differ between runs.

Step 4: the whole thing

Now assemble it with a config struct, an error path, and formatted output:

package main

import (
    "errors"
    "fmt"
    "sort"
    "strings"
    "unicode"
)

type WordCount struct {
    Word  string
    Count int
}

type Report struct {
    TotalWords  int
    UniqueWords int
    Top         []WordCount
}

type Options struct {
    TopN   int
    MinLen int
}

var ErrEmptyInput = errors.New("input contains no countable words")

func Analyze(text string, opts Options) (*Report, error) {
    if opts.TopN <= 0 {
        opts.TopN = 5
    }
    if opts.MinLen < 1 {
        opts.MinLen = 1
    }

    words := strings.FieldsFunc(strings.ToLower(text), func(r rune) bool {
        return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\''
    })

    counts := make(map[string]int)
    total := 0
    for _, w := range words {
        if len(w) < opts.MinLen {
            continue
        }
        total++
        counts[w]++
    }

    if total == 0 {
        return nil, fmt.Errorf("analyzing %d characters: %w", len(text), ErrEmptyInput)
    }

    ranked := make([]WordCount, 0, len(counts))
    for w, n := range counts {
        ranked = append(ranked, WordCount{Word: w, Count: n})
    }
    sort.Slice(ranked, func(i, j int) bool {
        if ranked[i].Count != ranked[j].Count {
            return ranked[i].Count > ranked[j].Count
        }
        return ranked[i].Word < ranked[j].Word
    })

    if len(ranked) > opts.TopN {
        ranked = ranked[:opts.TopN]
    }

    return &Report{TotalWords: total, UniqueWords: len(counts), Top: ranked}, nil
}

func (r *Report) String() string {
    var b strings.Builder
    fmt.Fprintf(&b, "%d words, %d unique\n", r.TotalWords, r.UniqueWords)
    for i, wc := range r.Top {
        bar := strings.Repeat("#", wc.Count)
        fmt.Fprintf(&b, "%d. %-12s %-6s %d\n", i+1, wc.Word, bar, wc.Count)
    }
    return b.String()
}

func main() {
    text := `Go is expressive, concise, clean, and efficient. Its concurrency
    mechanisms make it easy to write programs that get the most out of
    multicore machines, while its novel type system enables flexible and
    modular program construction. Go compiles quickly to machine code yet
    has the convenience of garbage collection and the power of run-time
    reflection. It is a fast, statically typed, compiled language that
    feels like a dynamically typed, interpreted language.`

    report, err := Analyze(text, Options{TopN: 6, MinLen: 4})
    if err != nil {
        fmt.Println("error:", err)
        return
    }
    fmt.Print(report)

    if _, err := Analyze("!!! ... ???", Options{}); err != nil {
        fmt.Println("\nempty input ->", err)
        fmt.Println("is ErrEmptyInput:", errors.Is(err, ErrEmptyInput))
    }
}

Walk through what's in there, because it's the course so far in one file:

  • Options with defaults filled in — the zero value is usable, so Analyze(text, Options{}) works (module 5).
  • A sentinel error wrapped with context — %w plus errors.Is at the call site (module 7).
  • A pointer result and nil on the error path — no typed-nil trap (module 6).
  • String() on *Report so fmt.Print(report) just works (module 6), built with a strings.Builder rather than += (module 4).
  • fmt.Fprintf to a writer rather than printing directly (module 6).

Making it a real CLI

The one piece the browser can't run — reading a file and flags:

package main

import (
    "flag"
    "fmt"
    "os"
)

func main() {
    topN := flag.Int("top", 10, "how many words to show")
    minLen := flag.Int("min", 3, "ignore words shorter than this")
    flag.Parse()

    if flag.NArg() < 1 {
        fmt.Fprintln(os.Stderr, "usage: wordfreq [-top N] [-min N] <file>")
        os.Exit(2)
    }

    data, err := os.ReadFile(flag.Arg(0))
    if err != nil {
        fmt.Fprintf(os.Stderr, "reading input: %v\n", err)
        os.Exit(1)
    }

    report, err := Analyze(string(data), Options{TopN: *topN, MinLen: *minLen})
    if err != nil {
        fmt.Fprintf(os.Stderr, "analyzing: %v\n", err)
        os.Exit(1)
    }

    fmt.Print(report)
}

Three conventions in there worth keeping: errors go to os.Stderr, a failing program exits non-zero (2 for usage, 1 for runtime failure), and flag gives you -h for free.

Because Analyze takes a string and returns a value rather than printing, it's testable without any of this — the CLI is a thin shell around a library. That separation is the single most useful structural habit in Go programs.

Your turn

Complete topWords: count the words, then return the top 3 as "word:count" strings, sorted by count descending and word ascending:

[go:3 is:2 fast:1]

package main

import (
    "fmt"
    "sort"
    "strings"
)

func topWords(text string, n int) []string {
    // count the fields, rank them by count desc then word asc,
    // and return the top n as "word:count"
}

func main() {
    fmt.Println(topWords("go is fast go is go", 3))
}

package main

import (
    "fmt"
    "sort"
    "strings"
)

func topWords(text string, n int) []string {
    counts := make(map[string]int)
    for _, w := range strings.Fields(text) {
        counts[w]++
    }

    words := make([]string, 0, len(counts))
    for w := range counts {
        words = append(words, w)
    }
    sort.Slice(words, func(i, j int) bool {
        if counts[words[i]] != counts[words[j]] {
            return counts[words[i]] > counts[words[j]]
        }
        return words[i] < words[j]
    })

    if len(words) > n {
        words = words[:n]
    }

    out := make([]string, 0, len(words))
    for _, w := range words {
        out = append(out, fmt.Sprintf("%s:%d", w, counts[w]))
    }
    return out
}

func main() {
    fmt.Println(topWords("go is fast go is go", 3))
}

Next capstone: doing many things at once, correctly.

← Previous Next →