examples: update lorem.v with a Markov algorithm for better text generation (#26085)

mike-ward · web-flow · commit 6af88dcd7349 · 2025-12-23T12:19:07.000+02:00
diff --git a/examples/lorem.v b/examples/lorem.v
@@ -1,166 +1,253 @@
+/*
+Random Markov Text Generator
+
+This program generates pseudo-random text using a Markov chain built from
+one of several embedded corpora. It produces structured output in the
+form of paragraphs and sentences, with configurable parameters for:
+
+- Markov order (n-gram size)
+- Words per sentence
+- Sentences per paragraph
+- Paragraph count
+- Optional seed phrases and RNG seed
+- Optional corpus selection
+
+Features:
+
+- Five built-in seed phrases, randomly chosen if no seed is provided
+- Paragraphs and sentences with ±20% variability in lengths
+- Automatic reseeding from corpus if seed phrases do not exist in the model
+- Fully self-contained; no external corpus files required
+- Can be run with no parameters and produces readable, multi-paragraph text
+
+Usage:
+
+    v run lorem.v [--options]
+    ./lorem [--options]
+
+Example:
+
+    ./lorem -order 2 -words 12 -sentences 4 -paragraphs 3 -corpus poe
+*/
 import rand
-import strings
 import flag
 import os
 
-// Features:
-// - All parameters configurable via flags
-// - Reproducible output via --seed
-// - No invalid punctuation sequences
-// - Em-dashes never near sentence edges
+// ---------------- Embedded Corpora ----------------
+
+const corpora = {
+	'lorem':  lorem_corpus
+	'poe':    poe_corpus
+	'darwin': darwin_corpus
+	'bard':   shakespeare_corpus
+}
 
-// ---------------- Configuration ----------------
+const lorem_corpus = '
+lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor
+incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis
+nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore
+eu fugiat nulla pariatur Excepteur sint occaecat cupidatat non proident sunt
+in culpa qui officia deserunt mollit anim id est laborum
+'
+
+const poe_corpus = '
+once upon a midnight dreary while I pondered weak and weary
+over many a quaint and curious volume of forgotten lore
+while I nodded nearly napping suddenly there came a tapping
+as of someone gently rapping at my chamber door
+'
+
+const darwin_corpus = '
+when we look to the individuals of the same variety or sub variety of our
+older cultivated plants and animals one of the first points which strikes
+us is that they generally differ much more from each other than do the
+individuals of any one species or variety in a state of nature
+'
+
+const shakespeare_corpus = '
+to be or not to be that is the question
+all the worlds a stage and all the men and women merely players
+the lady doth protest too much methinks
+a rose by any other name would smell as sweet
+et tu brute
+if music be the food of love play on
+now is the winter of our discontent
+we are such stuff as dreams are made on
+brevity is the soul of wit
+some are born great some achieve greatness and some have greatness thrust upon them
+cry havoc and let slip the dogs of war
+all that glisters is not gold
+the fault dear brutus is not in our stars but in ourselves
+to thine own self be true
+lord what fools these mortals be
+shall i compare thee to a summers day
+'
+
+// ---------------- Seed Phrases ----------------
+
+const seed_phrases = [
+	'in the beginning',
+	'once upon a time',
+	'it was the first',
+	'when we consider',
+	'there was a moment',
+]
+
+// ---------------- Main ----------------
 
-pub struct LoremCfg {
-	paragraphs              int = 3
+fn main() {
+	mut fp := flag.new_flag_parser(os.args[1..])
+	fp.application('lorem')
+	fp.version('1.0')
+	fp.description('Random text generator using a Markov chain')
+
+	order := fp.int('order', `o`, 2, 'Markov order [default: 2]')
+	words_per_sentence := fp.int('words', `w`, 10, 'Words per sentence [default: 10]')
+	sentences_per_paragraph := fp.int('sentences', `s`, 5, 'Sentences per paragraph [default: 5]')
+	paragraphs := fp.int('paragraphs', `p`, 3, 'Paragraph count [default: 3]')
+	corpus_name := fp.string('corpus', `c`, 'lorem', 'Corpus name (lorem, poe, darwin, bard) [default: lorem')
+	seed_text := fp.string('seed', `S`, '', 'Seed phrase (random if omitted)')
+	rng_seed := fp.int('rngseed', `r`, 0, 'RNG seed (0 = non-deterministic)')
+
+	fp.finalize() or {
+		eprintln(err)
+		return
+	}
+
+	text := generate_text(
+		order:                   order
+		words_per_sentence:      words_per_sentence
+		sentences_per_paragraph: sentences_per_paragraph
+		paragraphs:              paragraphs
+		corpus_name:             corpus_name
+		seed_text:               seed_text
+		rng_seed:                rng_seed
+	)
+
+	println(text)
+}
+
+struct LoremCfg {
+	order                   int = 2
 	words_per_sentence      int = 10
 	sentences_per_paragraph int = 5
-	commas_per_sentence     int = 2
-	seed                    int
+	paragraphs              int = 3
+	corpus_name             string
+	seed_text               string
+	rng_seed                int
 }
 
-// ---------------- Generator --------------------
-
-pub fn generate_lorem(cfg LoremCfg) string {
-	lorem_words := ('lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod ' +
-		'tempor incididunt ut labore et dolore magna aliqua enim ad minim veniam ' +
-		'quis nostrud exercitation ullamco laboris nisi aliquip ex ea').split(' ')
-
-	comma_marks := ['‚', '،', '，', ',', '﹐']
-	sentence_marks := ['.', '…', '。', '!', '?']
+// ---------------- Text Generation ----------------
 
-	if cfg.seed != 0 {
-		rand.seed([u32(cfg.seed), u32(cfg.seed ^ 0x9e3779b9)])
+fn generate_text(cfg LoremCfg) string {
+	if cfg.rng_seed != 0 {
+		rand.seed([u32(cfg.rng_seed)])
 	}
 
-	mut out := strings.new_builder(2048)
+	seed := match cfg.seed_text != '' {
+		true { cfg.seed_text }
+		else { random_seed_phrase() }
+	}
 
-	for p in 0 .. cfg.paragraphs {
-		sentence_count := vary(cfg.sentences_per_paragraph)
+	corpus := select_corpus(cfg.corpus_name)
+	tokens := tokenize(corpus)
 
-		for _ in 0 .. sentence_count {
-			word_count := vary(cfg.words_per_sentence)
+	if tokens.len <= cfg.order {
+		eprintln('corpus too small for selected order')
+		return ''
+	}
 
-			expected_breaks := if cfg.words_per_sentence > 0 {
-				(word_count * cfg.commas_per_sentence) / cfg.words_per_sentence
-			} else {
-				0
-			}
+	model := build_markov(tokens, cfg.order)
 
-			mut inserted_breaks := 0
-			mut last_had_splitter := false
-			min_dash_offset := 2
+	mut state := tokenize(seed)
+	if state.len < cfg.order {
+		start := rand.intn(tokens.len - cfg.order) or { 0 }
+		state = tokens[start..start + cfg.order].clone()
+	}
 
-			for w in 0 .. word_count {
-				word := lorem_words[rand.intn(lorem_words.len) or { 0 }]
+	mut out := []string{}
 
-				if w == 0 {
-					out.write_string(capitalize(word))
-				} else {
-					out.write_string(' ')
-					out.write_string(word)
-				}
+	for pi in 0 .. cfg.paragraphs {
+		if pi != 0 {
+			out << '\n\n'
+		}
+		sentences := vary(cfg.sentences_per_paragraph, 1)
 
-				is_last := w == word_count - 1
-
-				// ---- Clause splitters
-				if !is_last && inserted_breaks < expected_breaks
-					&& (w + 1) * expected_breaks / word_count > inserted_breaks {
-					use_strong := rand.intn(6) or { 0 } == 0
-
-					if use_strong {
-						if w >= min_dash_offset && w < word_count - 1 - min_dash_offset {
-							out.write_string('—')
-						} else {
-							out.write_string(';')
-						}
-					} else {
-						out.write_string(comma_marks[rand.intn(comma_marks.len) or { 0 }])
-					}
-
-					inserted_breaks++
-					last_had_splitter = true
+		for si in 0 .. sentences {
+			if si != 0 {
+				out << ' '
+			}
+			words := vary(cfg.words_per_sentence, 3)
+			mut sentence := []string{}
+
+			for _ in 0 .. words {
+				key := state.join('\u0001')
+				nexts := model[key] or {
+					start := rand.intn(tokens.len - cfg.order) or { 0 }
+					state = tokens[start..start + cfg.order].clone()
 					continue
 				}
 
-				last_had_splitter = false
+				next := nexts[rand.intn(nexts.len) or { 0 }]
+				sentence << next
 
-				// ---- Sentence terminator
-				if is_last {
-					if last_had_splitter {
-						out.write_string('.')
-					} else {
-						out.write_string(sentence_marks[rand.intn(sentence_marks.len) or { 0 }])
-					}
-				}
+				state = state[1..].clone()
+				state << next
 			}
 
-			out.write_string(' ')
-		}
-
-		if p < cfg.paragraphs - 1 {
-			out.write_string('\n\n')
+			if sentence.len > 0 {
+				out << sentence.join(' ').capitalize()
+				out << '.'
+			}
 		}
 	}
 
-	return out.str()
+	return out.join('')
 }
 
-// ---------------- Utilities --------------------
+// ---------------- Utilities ----------------
 
-fn vary(base int) int {
-	if base <= 1 {
-		return 1
+fn vary(base int, min int) int {
+	delta := int(f32(base) * 0.2)
+	if delta == 0 {
+		return base
 	}
-	delta := base / 5
-	r := rand.intn(delta * 2 + 1) or { 0 }
-	return clamp(base + r - delta, 1, 10_000)
+	offset := rand.intn(delta * 2 + 1) or { 0 } - delta
+	val := base + offset
+	return if val < min { min } else { val }
 }
 
-fn capitalize(s string) string {
-	if s.len == 0 {
-		return s
+fn select_corpus(name string) string {
+	if name != '' {
+		if corpus := corpora[name] {
+			return corpus
+		}
+		eprintln('unknown corpus: ${name}')
+		exit(1)
 	}
-	return s[..1].to_upper() + s[1..]
-}
 
-fn clamp(v int, min int, max int) int {
-	if v < min {
-		return min
-	}
-	if v > max {
-		return max
-	}
-	return v
+	keys := corpora.keys()
+	key := keys[rand.intn(keys.len) or { 0 }]
+	return corpora[key]
 }
 
-// ---------------- CLI --------------------------
-
-fn main() {
-	mut fp := flag.new_flag_parser(os.args)
-	fp.application('lorem')
-	fp.description('Lorem Ipsum generator with structured punctuation')
-	fp.skip_executable()
-
-	paragraphs := fp.int('paragraphs', `p`, 3, 'number of paragraphs')
-	words := fp.int('words', `w`, 10, 'approx words per sentence')
-	sentences := fp.int('sentences', `s`, 5, 'approx sentences per paragraph')
-	commas := fp.int('commas', `c`, 2, 'comma density per sentence')
-	seed := fp.int('seed', 0, 0, 'RNG seed (0 = deterministic default)')
+fn random_seed_phrase() string {
+	return seed_phrases[rand.intn(seed_phrases.len) or { 0 }]
+}
 
-	fp.finalize() or {
-		eprintln(err)
-		println(fp.usage())
-		return
-	}
+fn tokenize(text string) []string {
+	return text
+		.replace_each(['\n', ' ', '\t', ' '])
+		.split(' ')
+		.filter(it.len > 0)
+}
 
-	cfg := LoremCfg{
-		paragraphs:              paragraphs
-		words_per_sentence:      words
-		sentences_per_paragraph: sentences
-		commas_per_sentence:     commas
-		seed:                    seed
+fn build_markov(tokens []string, order int) map[string][]string {
+	mut model := map[string][]string{}
+	for i in 0 .. tokens.len - order {
+		key := tokens[i..i + order].join('\u0001')
+		model[key] << tokens[i + order]
 	}
-
-	println(generate_lorem(cfg))
+	return model
 }
