"""Sampling from a base model: the demonstrations of Section 6 (demo-base-model and the five after it) and step 8 on the slide on hallucinations (Section 8). Sends raw text to LM Studio's local server (/v1/completions, no chat template) and prints the continuation token by token. Only the temperature changes the model's probabilities unless top-p or top-k is set (LM Studio's own defaults, top-k 40, top-p 0.95, min-p 0.05 and a repeat penalty, are switched off). A stop string is applied here, as an application does: the model writes it, the program shows it in red and cuts the text there. Python 3, standard library only. python3 base.py Then type: 1 ... 8 the steps, in the order of the slides: 1 an article opening 5 after the cutoff 2 three samples 6 dialogue, no stop 3 temperature, top-p 7 dialogue with stop 4 memorised texts 8 a birthday, three runs 7 your question step 7 with your question in the dialogue: the model plays the assistant any other text a prompt; write \\n for a new line (empty line) the last prompt again /t 0.7 temperature /n 40 maximum number of new tokens /p 0.9 top-p (1 = off) /k 5 top-k (0 = off) /stop User: stop string /stop no stop string /clear clear the screen /q quit (or Ctrl-D) and unload the model; Ctrl-C stops a running generation """ import json import os import shutil import subprocess import sys import time import urllib.request try: import readline # arrow keys and history in input() except ImportError: pass SERVER = "http://localhost:1234" MODEL = os.environ.get("MODEL", "olmo-2-1124-7b") KOLMOGOROV = "Andrey Kolmogorov (1903–1987) was a Soviet mathematician who" DIALOGUE = ("A conversation between a user and an assistant who answers briefly.\n\n" "User: What is the derivative of sin x?\nAssistant: cos x.\n\n" "User: Is 91 a prime number?\nAssistant: No, 91 = 7 × 13.\n\n" "User: {question}\nAssistant:") MIT = "Permission is hereby granted, free of charge, to any person obtaining a copy" SHAKESPEARE = ("William Shakespeare (c. 23 April 1564 – 23 April 1616) was an English playwright, " "poet and actor. He is widely regarded as") # the Wikipedia article, October 2026 KALAI = "What is the birthday of Adam Tauman Kalai? If you know, just respond with DD-MM." # the example on the slide on hallucinations EURO = "BERLIN, July 14, 2024 — The final of Euro 2024 at the Olympiastadion ended" # after the cutoff, December 2023 # step -> list of (label, prompt, temperature, max_tokens, stop, top_p), in the order of the slides STEPS = { "1": [("temperature 1", KOLMOGOROV, 1.0, 80, None, 1.0)], "2": [("run %d of 3, temperature 1" % k, KOLMOGOROV, 1.0, 80, None, 1.0) for k in (1, 2, 3)], "3": [("temperature 0, run 1", KOLMOGOROV, 0.0, 80, None, 1.0), ("temperature 0, run 2", KOLMOGOROV, 0.0, 80, None, 1.0), ("temperature 2", KOLMOGOROV, 2.0, 60, None, 1.0), ("temperature 2, top-p 0.9", KOLMOGOROV, 2.0, 60, None, 0.9), ("temperature 1, max tokens 10", KOLMOGOROV, 1.0, 10, None, 1.0)], "4": [("Wikipedia, temperature 0", SHAKESPEARE, 0.0, 40, None, 1.0), ("MIT License, temperature 0", MIT, 0.0, 120, None, 1.0)], "5": [("after the cutoff, temperature 0", EURO, 0.0, 70, None, 1.0), ("after the cutoff, temperature 1, top-p 0.9", EURO, 1.0, 70, None, 0.9), ("after the cutoff, temperature 1, top-p 0.9", EURO, 1.0, 70, None, 0.9)], "6": [("no stop string", DIALOGUE.format(question="What is the integral of 1/x?"), 1.0, 80, None, 1.0)], "7": [("stop string \"\\nUser:\"", DIALOGUE.format(question="What is the integral of 1/x?"), 1.0, 80, "\nUser:", 1.0)], "8": [("run %d of 3, temperature 1, stop string \"\\nUser:\"" % k, DIALOGUE.format(question=KALAI), 1.0, 60, "\nUser:", 1.0) for k in (1, 2, 3)], } BOLD, DIM, GREEN, RED, RESET = "\033[1m", "\033[2m", "\033[32m", "\033[31m", "\033[0m" def get(path): with urllib.request.urlopen(SERVER + path, timeout=3) as r: return json.load(r) def lms(): return shutil.which("lms") or os.path.expanduser("~/.lmstudio/bin/lms") def ensure_model(): """Start LM Studio's server and load the model if they are not running yet.""" try: get("/v1/models") except OSError: print(DIM + "starting the LM Studio server ..." + RESET) subprocess.run([lms(), "server", "start"], check=True) loaded = [m["id"] for m in get("/api/v0/models")["data"] if m.get("state") == "loaded"] if MODEL not in loaded: print(DIM + "loading %s ..." % MODEL + RESET) subprocess.run([lms(), "load", MODEL, "--yes"], check=True) def complete(prompt, temperature, max_tokens, stop, top_p=1.0, top_k=0): body = {"model": MODEL, "prompt": prompt, "temperature": temperature, "max_tokens": max_tokens, "stream": True, "top_p": top_p, "top_k": top_k, "min_p": 0, "repeat_penalty": 1} req = urllib.request.Request(SERVER + "/v1/completions", json.dumps(body).encode(), {"Content-Type": "application/json"}) print(BOLD + prompt + RESET + GREEN, end="", flush=True) start, first, n, finish = time.time(), None, 0, "?" text, shown = "", 0 # the stop string is checked here, so hold back its possible start try: with urllib.request.urlopen(req) as r: for line in r: line = line.decode().strip() if not line.startswith("data: ") or line == "data: [DONE]": continue choice = json.loads(line[6:])["choices"][0] if choice.get("text"): first = first or time.time() n += 1 text += choice["text"] cut = text.find(stop) if stop else -1 if cut >= 0: print(text[shown:cut] + RED + stop + RESET + DIM + " <- stop string: the program cuts here" + RESET, end="", flush=True) shown, finish = len(text), "stop string" break safe = len(text) - (len(stop) - 1 if stop else 0) if safe > shown: print(text[shown:safe], end="", flush=True) shown = safe finish = choice.get("finish_reason") or finish except KeyboardInterrupt: finish = "interrupted" if shown < len(text): print(text[shown:], end="") rate = "" if not first or n < 2 else ", %.0f tokens/s" % ((n - 1) / (time.time() - first)) print(RESET + "\n" + DIM + "[%d tokens%s; finish: %s]" % (n, rate, finish) + RESET + "\n") def main(): ensure_model() try: repl() finally: print(DIM + "unloading %s ..." % MODEL + RESET) subprocess.run([lms(), "unload", MODEL], capture_output=True) def repl(): print(__doc__.split("Then type:")[1]) t, n, p, k, stop, last = 1.0, 80, 1.0, 0, None, None while True: try: line = input(DIM + "[t=%g n=%d p=%g k=%d stop=%s] " % (t, n, p, k, json.dumps(stop)) + RESET + "> ") except (EOFError, KeyboardInterrupt): print() return line = line.strip() if line in ("/q", "/quit"): return if line == "/clear": print("\033[2J\033[3J\033[H", end="", flush=True) continue if line.startswith("7 "): print(DIM + "--- step 7 with your question, stop string \"\\nUser:\"" + RESET) last = DIALOGUE.format(question=line[2:].strip()) complete(last, 1.0, 80, "\nUser:") elif line in STEPS: for label, prompt, st, sn, sstop, sp in STEPS[line]: print(DIM + "--- step %s: %s" % (line, label) + RESET) complete(prompt, st, sn, sstop, sp) last = STEPS[line][-1][1] elif line.startswith("/t "): t = float(line[3:]) elif line.startswith("/n "): n = int(line[3:]) elif line.startswith("/p "): p = float(line[3:]) elif line.startswith("/k "): k = int(line[3:]) elif line.startswith("/stop"): stop = line[6:].replace("\\n", "\n") or None elif line.startswith("/"): print(__doc__.split("Then type:")[1]) elif line or last: last = line.replace("\\n", "\n") if line else last complete(last, t, n, stop, p, k) if __name__ == "__main__": sys.exit(main())