Examples

A handful of example programs demonstrating various features of Pointless. Some examples (self-hosting tokenizer and stack-based language vm) span multiple source files.

100 Doors (source)
100Doors.ptls
----------------------------------------------------------
-- Solution to the '100 doors' problem
-- http://rosettacode.org/wiki/100_doors

output =
  range(1, 100)
  |> map(visit(100))
  |> println

----------------------------------------------------------

toggle(state) =
  if state == Closed then Open else Closed

----------------------------------------------------------
-- Door state on iteration i is recursively
-- defined in terms of previous door state

visit(i, index) = cond {
  case (i == 0) Closed
  case (index % i == 0) toggle(lastState)
  else lastState
} where lastState = visit(i - 1, index)
$ bin/pointless examples/100Doors/100Doors.ptls

[Open, Closed, Closed, Open, Closed, Closed, Closed, Closed, Open, Closed ...]
        

99 Bottles of Beer (source)
beer.ptls
-----------------------------------------------------------
-- Print the lyrics to the song '99 bottles of beer'

output =
  range(99, 1)
  |> map(showBeer)
  |> printLines

showBeer(n) =
  format("{} of beer on the wall!\n", [showBottle(n)]) +
  format("{} of beer!\n", [showBottle(n)]) +
  "You take one down, pass it around\n" +
  nextBeer(n - 1)

nextBeer(n) =
  if n == 0 then "No more bottles of beer on the wall!"
  else format("{} of beer on the wall!\n", [showBottle(n)])

-----------------------------------------------------------
-- Get appropriate singular / plural form of 'n bottle(s)'

showBottle(n) =
  format("{} {}", [n, bottleStr])
  where bottleStr = if n == 1 then "bottle" else "bottles"
$ bin/pointless examples/beer/beer.ptls

...

2 bottles of beer on the wall!
2 bottles of beer!
You take one down, pass it around
1 bottle of beer on the wall!

1 bottle of beer on the wall!
1 bottle of beer!
You take one down, pass it around
No more bottles of beer on the wall!

Chart (source)
chart.ptls
-----------------------------------------------------------
-- Given a list of numerical values, print an ascii bar-
-- chart with dimensions (in terms of characters) 
-- height x length(values), where the largest value takes
-- up the full height of the chart
-- Negative values get truncated to zero 

scale(height, values) =
  if values == Empty then ""
  else
    values
    |> normalize(height)
    |> getRows(height)
    |> join("\n")

-----------------------------------------------------------
-- Get list of scaled values so that max(scaled) = height

normalize(height, values) =
  values
  |> map(mul(height / maximum(values)))
  |> map(max(0))

-----------------------------------------------------------
-- Build up the chart row-by-row starting with the hightest
-- row index (from top to bottom)

getRows(height, values) =
  for row in reverse(range(0, height - 1))
  yield rowChars(row, values) |> join("")

rowChars(row, values) =
  values |> map(getBar(row))

-----------------------------------------------------------
-- BarHeight is the height that the value n projects
-- above the base of the current row (capped at max value
-- seven when the value spans the entire row height)
-- Use this value to index into the array of bar chars

getBar(row, n) = cond {
  case barHeight < 0 and row > 0  " "
  case barHeight < 0 and row == 0 "_"
  else bars[min(7, toInt(barHeight * 7))] 
} where {barHeight = (n - row); bars = toArray("▁▂▃▄▅▆▇█")}
-- chart.ptls doesn't have an output variable defined, so we can't execute it directly,
-- but we can import it to use its functions in other scripts, as seen in collatz.ptls

Collatz Sequence (source)
collatz.ptls
import "../chart/chart.ptls" as chart

-----------------------------------------------------------
-- Calculate each number in the collatz sequence for some
-- starting value, and ending when the value 1 is reached
-- Display these values in an ascii bar-chart with 8 rows

output =
  iterate(step, 175) -- sequence starts at 175
  |> takeUntil(eq(1))
  |> chart.scale(4)
  |> println

-----------------------------------------------------------
-- get the next number in a collatz sequence

step(n) =
  if n % 2 == 0 then n / 2 else n * 3 + 1
$ bin/pointless examples/collatz/collatz.ptls

                                    ▂         █                                  
                                  ▁ █       ▅ █▁                                 
                                ▃ █▁█▅  ▂ ▆ █▃██▁                                
▁▂▁▃▂▄▂▆▃▂▅▃▂▁▂▁▃▂▄▂▁▃▂▄▂▁▃▂▅▃▇▄█▅████▆▃█▅█▇█████▄▂▆▃▂▄▂▁▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁

Echo Lines (source)
echoLines.ptls
-----------------------------------------------------------
-- Read lines from stdin, and print each line along
-- with its line number, right-justified to 3 chars 

output =
  readLines
  |> enumerate
  |> map(format("{>3}: {}"))
  |> printLines
$ cat examples/echoLines/langs.txt | bin/pointless examples/echoLines/echoLines.ptls

  0: c++:
  1:   "The position you applied for is no longer available"
  2: 
  3: java:
  4:   "We sincerely appreciated your enthusiasm, outstanding
  5:   interview, impeccible resume, and engaging personality.
  6:   It is rare to find a candidate as qualified and capable
  7:   as yourself. That being said, we regret to inform you that..." 
  8: 
  9: python:
 10:   "... and we provide each of our software engineers with a standing desk",
 11:   "Actually, I'd prefer a normal desk if that's...",
 12:   "WE PROVIDE EACH OF OUR SOFTWARE ENGINEERS WITH A STANDING DESK" 
 13: 
 14: haskell:
 15:  "t o = whomSt :: so|ever| <*> it m $ ay conc . ern; it 
 16:   | is ++ m y !Pleasur (*e) ~~>> to >>= _ \i -> n fo (R m y o !! u)
 17:   | that -> [yo..u"
 18: 
 19: javascript:
 20:   "click here to learn all of ES6 in 20 seconds!
 21:   http://obliterating.the.coding.interview.io/"

Number Guessing (source)
guess.ptls
number = randRange(0, 1000)

getResponse(n) =
  format("{} is {}", [n, compareGuess(n)])

compareGuess(n) = cond {
  case n > number  "too high"
  case n < number  "too low"
  case n == number "correct!"
}

----------------------------------------------------------
-- A number guessing game (binary search)
-- User enters an integer guess between 0 and 1000
-- progarm reports whether guess is above or below the
-- target number - continues until correct guess is entered

output =
  readLines
  |> map(toInt)
  |> takeUntil(eq(number))
  |> map(getResponse)
  |> printLines
$ bin/pointless examples/guess/guess.ptls

500
500 is too high
250
250 is too low
375
375 is too high
312
312 is correct!

Game of Life (source)
life.ptls
-----------------------------------------------------------
-- Print 100 simulated states of conway's game of life
-- for a glider starting pattern
-- Print generation number along with cells

output =
  initCells
  |> iterate(updateCells)
  |> take(20)
  |> enumerate
  |> map(showPair)
  |> printFrames

-- need to concat results from printFrames to join the
-- commands into one big sequence

initCells =
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
  [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

width  = length(initCells[0])
height = length(initCells)

positions = 
  for y in range(0, height - 1)
  for x in range(0, width - 1)
  yield (x, y)

-----------------------------------------------------------
-- Update each cell in the grid according to its position,
-- and convert the resulting list back to a 2D array

updateCells(cells) =
  positions
  |> map(tick(cells))
  |> toNDArray((height, width))

-----------------------------------------------------------
-- Get the new value for a cell at a give position
-- based on the current cell values in the grid

tick(cells, pos) = toInt(survive or birth) where {
  survive = cells[y][x] == 1 and count in {2, 3}
  birth   = cells[y][x] == 0 and count == 3
  count   = getCount(x, y, cells)
  (x, y)  = pos
}

-----------------------------------------------------------
-- Get the number of live neighbors of a given position

getCount(x, y, cells) = sum(neighborVals) where {
  neighborVals = 
    for dx in deltas 
    for dy in deltas 
    when (dx != 0 or dy != 0)
    yield getNeighbor(x + dx, y + dy, cells) 
  deltas = [-1, 0, 1]
}

getNeighbor(x, y, cells) = cells[y % height][x % width]   

-----------------------------------------------------------
-- Print the board and generation number given pairs
-- of (gen, cells) from the enumerate function

showPair(pair) =
  format("{}\ngeneration: {}", [showCells(cells), gen])
  where (gen, cells) = pair

showCells(cells) =
  toList(cells)
  |> map(showRow)
  |> join("\n")

showRow(row) =
  format("|{}|", [map(showCell, toList(row)) |> join("")])

showCell(cell) =
  if cell == 1 then "*" else " "
$ bin/pointless examples/life/life.ptls

|                            |
|  *                         |
|   *                        |
| ***                        |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
generation: 0

|                            |
|                            |
| * *                        |
|  **                        |
|  *                         |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
generation: 1

|                            |
|                            |
|   *                        |
| * *                        |
|  **                        |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
|                            |
generation: 2

...

Permutations (source)
permutations.ptls
-----------------------------------------------------------
-- For a list of values, get a list of all permutations of
-- the items in the original list
-- (algorithm produces duplicate permutations for lists
-- containing duplicate elements)

permutations(list) = cond {
  case isEmpty(list) []
  case isEmpty(tail(list)) [list]
  else
    for elem in list
    for perm in permutations(delete(elem, list))
    yield [elem] ++ perm
}
  
-----------------------------------------------------------
  
delete(elem, list) = cond {
  case isEmpty(list) []
  case (head(list) == elem) tail(list)
  else [head(list)] ++ delete(elem, tail(list))
}

-----------------------------------------------------------

output =
  range(0, 2) -- this example uses the list [0, 1, 2]
  |> permutations
  |> println
$ bin/pointless examples/permutations/permutations.ptls

[[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]]

Quine (a self-printing program) (source)
quine.ptls
output = (s => println(s + show(toTuple([s]))))("output = (s => println(s + show(toTuple([s]))))")
$ bin/pointless examples/quine/quine.ptls

output = (s => println(s + show(toTuple([s]))))("output = (s => println(s + show(toTuple([s]))))")

Stack-Based Language VM (source)
factorsVM.ptls
import "stackVM.ptls" as stackVM

------------------------------------------------------------------------------

output =
  program
  |> stackVM.runGetOutput
  |> printLines

------------------------------------------------------------------------------

n = 80122

program = [
  Const(2),
  Store(0),            -- store value 2 in slot 0 (initial divisor)
  Const(n),
  Store(1),            -- store value n in slot 1 (initial dividend)
  Label("checkDone"),
  Load(1),             -- load dividend
  Const(1),
  Eq,                  -- compare dividend to value 1
  JmpIf("end"),        -- jump to end if dividend == 1
  Label("checkDiv"), 
  Load(1),             -- load dividend
  Load(0),             -- load divisor
  Mod,
  Const(0),
  Eq,                  -- compare dividend % dividor == 0 
  JmpIf("divisible"),
  Load(0),             -- load divisor
  Const(1),
  Add,                 -- increment divisor
  Store(0),            -- store new divisor
  Jmp("checkDiv"),     -- check end condition
  Label("divisible"),
  Load(0),             -- load divisor
  Print,               -- print divisor (current factor)
  Load(1),             -- load dividend
  Load(0),             -- load divisor
  Div,
  Store(1),            -- store dividend / divisor as new dividend
  Jmp("checkDone"),    -- check end condition
  Label("end"),
  Const("done"),
  Print,               -- print "done"
  Exit,
]
$ bin/pointless examples/stackVM/factorsVM.ptls

2
7
59
97
done
collatzVM.ptls
import "stackVM.ptls" as stackVM

------------------------------------------------------------------------------

output =
  program
  |> stackVM.runGetOutput
  |> printLines

------------------------------------------------------------------------------

n = 870

program = [
  Const(n),
  Label("startLoop"),
  Store(0),
  Load(0),
  Const(1),
  Eq,
  JmpIf("end"),
  Load(0),
  Print,
  Load(0),
  Const(2),
  Mod,
  Const(0),
  Eq,
  JmpIf("even"),
  Load(0),
  Const(3),
  Mul,
  Const(1),
  Add,
  Jmp("startLoop"),
  Label("even"),
  Load(0),
  Const(2),
  Div,
  Jmp("startLoop"),
  Label("end"),
  Load(0),
  Print,
  Const("done"),
  Print,
  Exit,
]
$ bin/pointless examples/stackVM/collatzVM.ptls

870
435.0
...
4.0
2.0
1.0
done
stackVM.ptls
export {
  runGetOutput
}

------------------------------------------------------------------------------

import "instructions.ptls" as instructions

------------------------------------------------------------------------------

vmFromInsts(program) = VM {
  slots  = zeroArray(8)
  insts  = toArray(instructions.convertJumps(program))
  index  = 0
  stack  = []
  outVal = None
}

------------------------------------------------------------------------------

runGetOutput(insts) =
  vmFromInsts(insts)
  |> iterate(compose(setOutput(None), eval))
  |> takeUntil(vm => vm.insts[vm.index] == Exit)
  |> map(vm => vm.outVal)
  |> filter(notEq(None))

------------------------------------------------------------------------------

eval(vm) = cond {
  case is(Const, inst)
    vm
    |> pushVal(arg)
    |> advance

  case is(Print, inst)
    vm
    |> popVals(1)
    |> setOutput(valAt(0, vm))
    |> advance

  case is(Load, inst)
    vm
    |> load(arg)
    |> advance

  case is(Store, inst)
    vm
    |> store(arg)
    |> popVals(1)
    |> advance

  case is(JmpIf, inst)
    vm
    |> popVals(1)
    |> jumpIf(valAt(0, vm), arg)

  case is(Jmp, inst)
    vm
    |> jumpIf(true, arg)

  case is(Eq,  inst) vm |> binaryOp(eq)
  case is(Add, inst) vm |> binaryOp(add)
  case is(Sub, inst) vm |> binaryOp(sub)
  case is(Mul, inst) vm |> binaryOp(mul)
  case is(Div, inst) vm |> binaryOp(div)
  case is(Mod, inst) vm |> binaryOp(mod)

} where {
  inst = vm.insts[vm.index]
  arg = unwrap(inst)
}

------------------------------------------------------------------------------

advance(vm)        = vm with $.index += 1
pushVal(arg, vm)   = vm with $.stack = [arg] ++ vm.stack
popVals(n, vm)     = vm with $.stack = drop(n, vm.stack)
setOutput(val, vm) = vm with $.outVal = val
store(arg, vm)     = vm with $.slots[arg] = head(vm.stack)
load(arg, vm)      = pushVal(vm.slots[arg], vm)

jumpIf(pred, arg, vm) =
  if pred
  then vm with $.index = arg
  else advance(vm) 

valAt(n, vm) = at(n, vm.stack)

binaryOp(op, vm) =
 vm
 |> popVals(2)
 |> pushVal(op(valAt(0, vm), valAt(1, vm)))
 |> advance
-- see examples above
instructions.ptls
export {
  showInst, makeInstArray
}

------------------------------------------------------------------------------

showInst(inst) =
 format("[ {<5} ] {}", [getLabel(inst), argOrBlank(inst)])

argOrBlank(inst) =
  if is(PtlsTuple, inst) then unwrap(inst) else ""

------------------------------------------------------------------------------

convertJumps(insts) =
  insts
  |> filter(notIs(Label))
  |> map(convertJump(inds))
  where inds = getLabelInds(insts)

------------------------------------------------------------------------------

convertJump(inds, inst) = cond {
  case is(Jmp, inst) Jmp(inds[unwrap(inst)])
  case is(JmpIf, inst) JmpIf(inds[unwrap(inst)])
  else inst
}

------------------------------------------------------------------------------

getLabelInds(insts) =
  insts
  |> reduce(scanInst, (0, {})) -- keep track of (currentIndex, indexMap)
  |> at(1) -- return index map

------------------------------------------------------------------------------

scanInst(pair, inst) = 
  if not is(Label, inst)
  then (ind + 1, inds)
  else (ind, newInds)
  where {
    newInds = inds with $[unwrap(inst)] = ind
    (ind, inds) = pair
  }
-- see examples above

Self-Hosting Tokenizer (source)
tokenize.ptls
import "location.ptls" as location
import "symbols.ptls" as symbols

------------------------------------------------------------------------------
-- For tokenizer tok with chars c0, c1, c2 ... cn
-- and funcs f0, f1, f2, ... fn
-- return true if n >= m and all([f0(c0), f1(c1), ...])

matchChars(funcs, tok) =
  if length(pairs) < length(funcs) then false
  else
    pairs
    |> map(pair => (func(char) where (func, char) = pair))
    |> all
  where pairs = zip(funcs, tok.chars)

------------------------------------------------------------------------------
-- New tokenizer object

new(text) = Tokenizer {
  chars    = split("", text)
  locs     = location.fromChars("path", chars)
  stack    = []
  tokens   = []
  tokenLoc = head(locs) -- where the current token started
}

------------------------------------------------------------------------------
-- Move forward one char, push char onto stack

advance(tok) =
  tok with {
    $.chars = tail(tok.chars)
    $.stack = [head(tok.chars)] ++ tok.stack
    $.locs  = tail(tok.locs)
  }

------------------------------------------------------------------------------
-- Skip chars (don't push to stack)

skipVal(tok) = advance(tok) with {
  -- keep stack of tok before advance (leaves out current char)
  $.stack = tok.stack
}

------------------------------------------------------------------------------
-- Push chars to stack as long as matchChars(funcs)

advanceWhile(funcs, tok) =
  iterate(advance, tok)
  |> dropWhile(matchChars(funcs))
  |> head

------------------------------------------------------------------------------
-- Skip chars (don't push to stack) as long as matchChars(funcs)

skipWhile(funcs, tok) =
  iterate(skipVal, tok)
  |> dropWhile(matchChars(funcs))
  |> head

------------------------------------------------------------------------------
-- No more chars to parse

isEOF(tok) = tok.chars == Empty

------------------------------------------------------------------------------
-- Get token string from stack of tokens

tokVal(tok) =
  tok.stack
  |> reverse -- pushing chars reversed order - reverse again to correct
  |> join("")

------------------------------------------------------------------------------
-- Push new token to token stack

makeToken(labelFunc, tok) = tok with {
  $.tokens   = [token] ++ tok.tokens
  $.stack    = []                       -- clear stack
  $.tokenLoc = head(tok.locs)           -- set loc for next token

} where {
  -- add label to token object (Name {...})
  token = wrapObject(labelFunc(val), {value = val; loc = tok.tokenLoc})
  val = tokVal(tok)
}

------------------------------------------------------------------------------
-- Find matching token type for upcoming chars

isBlank        = matchChars([eq("_")])
isComment      = matchChars([eq("-"), eq("-")])
isInt          = matchChars([inFunc(digits)])
isFloat        = matchChars([eq("."), inFunc(digits)])
isName         = matchChars([inFunc(lowers)])
isLabel        = matchChars([inFunc(uppers)])
isCustomField  = matchChars([eq("."), inFunc(alphas)])
isLangField    = matchChars([eq("."), eq("!"), inFunc(alphas)])
isOpSym        = matchChars([inFunc(symbols.opSymChars)])
isWhitespace   = matchChars([eq(" ")])
isNewline      = matchChars([eq("\n")])
isSeparator    = matchChars([inFunc(symbols.separators)])
isLeftSym      = matchChars([inFunc(symbols.leftSyms)])
isRightSym     = matchChars([inFunc(symbols.rightSyms)])
isString       = matchChars([eq("\"")])

------------------------------------------------------------------------------
-- Make a new token and push to token stack
-- must have not isEOF(tok)
-- throws TokenizerError

getToken(tok) = cond {

  case isBlank(tok)
    tok
    |> skipVal -- skip blank char
    |> makeToken(const(Blank))

  ----------------------------------------------------------------------------

  case isComment(tok)
    tok
    |> advanceWhile([eq("-")]) -- take all dashes
    |> advanceWhile([notEq("\n")]) -- comment extends to the end of line
    |> makeToken(const(Comment))

  ----------------------------------------------------------------------------

  case isWhitespace(tok)
    tok
    |> skipWhile([eq(" ")]) -- combine repeated spaces into one token
    |> makeToken(const(Whitespace))

  ----------------------------------------------------------------------------

  case isNewline(tok)
    tok
    |> skipWhile([eq("\n")]) -- combine repeated newlines into one token
    |> makeToken(const(Newline))

  ----------------------------------------------------------------------------

  case isSeparator(tok)
    tok
    |> advance -- take separator char
    |> makeToken(getIndex(symbols.separators))

  ----------------------------------------------------------------------------

  case isName(tok)
    tok
    |> advanceWhile([inFunc(alNums)]) -- take all alphanumeric chars
    -- return matching keyword token if name is a keyword
    |> makeToken(getDefault(symbols.keywords, Name))

  ----------------------------------------------------------------------------

  case isCustomField(tok)
    tok
    |> skipVal -- take '.'
    |> advanceWhile([inFunc(alNums)])
    |> makeToken(const(Field))

  ----------------------------------------------------------------------------

  case isLangField(tok)
    tok
    |> skipVal -- take '.'
    |> advance -- take '!'
    |> advanceWhile([inFunc(alNums)])
    |> makeToken(const(Field))

  ----------------------------------------------------------------------------

  case isLabel(tok)
    tok
    |> advanceWhile([inFunc(alNums)]) -- take all alphanumeric chars
    |> makeToken(const(Label))

  ----------------------------------------------------------------------------

  case isString(tok)
    tok
    |> skipVal -- skip opening '"'
    |> getStringChars
    |> skipVal -- skip closing '"'
    |> makeToken(const(String))

  ----------------------------------------------------------------------------

  case isInt(tok)
    tok
    |> advanceWhile([inFunc(digits)]) -- take all leading digits
    -- if there's a decimal point followed by more digits, take those too
    |> (tok => if isFloat(tok) then getFloatDigits(tok) else tok)
    |> makeToken(const(Number))

  ----------------------------------------------------------------------------

  case isFloat(tok)
    tok
    |> getFloatDigits(tok) -- get decimal point and following digits
    |> makeToken(const(Number))

  ----------------------------------------------------------------------------

  case isOpSym(tok)
    tok
    |> advanceWhile([inFunc(symbols.opSymChars)]) -- take all operator chars
    |> checkOpSym -- check that chars form a valid operator
    |> makeToken(getIndex(symbols.opSyms))

  ----------------------------------------------------------------------------
  -- leftSyms and rightSyms kept separate to accomodate Sub -> Neg conversion

  case isLeftSym(tok)
    tok
    |> advance -- take leftSym char
    |> makeToken(getIndex(symbols.leftSyms))

  ----------------------------------------------------------------------------

  case isRightSym(tok)
    tok
    |> advance -- take rightSym char
    |> makeToken(getIndex(symbols.rightSyms))

  ----------------------------------------------------------------------------
  -- leading chars don't form a valid token
  -- example: a non alphaNumeric char not in any operator or symbol like '&'

  else
    tokError(tok, message)
    where message = format("Unexpected symbol '{}'", [head(tok.chars)])

} requires not isEOF(tok)

------------------------------------------------------------------------------
-- Get chars inside string

getStringChars(tok) = cond {
  case isEOF(tok)
    -- reached end of file before closing quote
    tokError(tok, "Unmatched quote")

  case matchChars([eq("\n")], tok)
    -- reached end of line before closing quote
    tokError(tok, "Unmatched quote (must escape line breaks in string)")

  case matchChars([eq("\\")], tok)
    -- on backslash, move past backslash and escape char
    getStringChars(advance(advance(tok)))

  case matchChars([eq("\"")], tok)
    tok -- stop before taking closing quote

  else getStringChars(advance(tok))
}

------------------------------------------------------------------------------
-- Get decimal point and following digits

getFloatDigits(tok) =
  tok
  |> advance
  |> advanceWhile([inFunc(digits)])

------------------------------------------------------------------------------
-- Check that current token chars form valid operator

checkOpSym(tok) =
  if val in symbols.opSyms then tok
  else tokError(tok, format("Invalid operator '{}'", [val]))
  where val = tokVal(tok)

------------------------------------------------------------------------------
-- Throw TokenizerError

tokError(tok, message) = throw TokenizerError(fullMessage)
  where fullMessage = format(
    "{}\nTokenizer Error:\n\n{}\n{}\n\n{}",
    [sep, message, sep, location.showLoc(tok.tokenLoc)]
  )

sep = repeat("-") |> take(79) |> join("")

------------------------------------------------------------------------------
-- Get string representation for token

showTok(token) =
  format(
    "{>3}:{<2} [ {<12} ] {}",
    [line, col, getLabel(token), token.value]
  ) where (line, col) = at(0, token.loc)
 
------------------------------------------------------------------------------

startToks = toSet(
  vals(symbols.opSyms)
  ++ vals(symbols.separators)
  ++ vals(symbols.leftSyms)
  ++ vals(symbols.keywords)
)

endToks = toSet(
  vals(symbols.rightSyms)
  ++ [Name, Field, String, Number]
)

updateIsStart(isStart, token) = cond {
  case getLabel(token) in startToks true
  case getLabel(token) in endToks false
  else isStart
}

------------------------------------------------------------------------------

getTokens(tok) =
  tok
  |> iterate(getToken)
  |> drop(1)
  |> takeUntil(isEOF)
  |> map(tok => head(tok.tokens))

------------------------------------------------------------------------------

convert(tokens) =
  scan(updateIsStart, false, tokens)
  |> zip(tokens)
  |> map(convertSubExpr)
  |> getNeighborTruples
  |> map(convertSubPadding)

------------------------------------------------------------------------------
-- Scan tokens, keeping track of whether '-' corresponds to a negative
-- sign or subtraction operator, depending on whether they occur at the start
-- of an expression, and update (all '-' are initially parsed as subtraction)

convertSubExpr(pair) =
  if is(Sub, token) and isStart
  then wrapObject(Neg, token)
  else token
  where (token, isStart) = pair

------------------------------------------------------------------------------

getNeighborTruples(tokens) =
  zipN([tokens, [None] ++ tokens, drop(1, tokens)])

------------------------------------------------------------------------------
-- Special case: conver '-' to neg in 'a -b'
-- for examplem:
--   array = [1 2 -3] 

convertSubPadding(triple) =
  if is(Sub, current) and is(Whitespace, last) and is(Number, next)
  then wrapObject(Neg, current)
  else current
  where (current, last, next) = triple

------------------------------------------------------------------------------

text = readLines |> join("\n")

output =
  try
    new(text)
    |> getTokens
    |> convert
    |> map(showTok)
    |> eager
    |> printLines

  catch is(TokenizerError)
    err => println(unwrap(err))
$ cat examples/tokenizer/tokenizer.ptls | bin/pointless examples/tokenizer/tokenizer.ptls 

  1:1  [ Newline      ] 
  2:1  [ Import       ] import
  2:7  [ Whitespace   ] 
  2:8  [ String       ] location.ptls
  2:23 [ Whitespace   ] 
  2:24 [ As           ] as
  2:26 [ Whitespace   ] 
  2:27 [ Name         ] location
  2:35 [ Newline      ] 
  3:1  [ Import       ] import
  3:7  [ Whitespace   ] 
  3:8  [ String       ] symbols.ptls
  3:22 [ Whitespace   ] 
  3:23 [ As           ] as
  ...
location.ptls
export {fromChars, showLoc}

------------------------------------------------------------------------------

nextPos(pos, char) =
  if char == "\n" then (line + 1, 1) else (line, col + 1)
  where (line, col) = pos

------------------------------------------------------------------------------

makeLoc(lines, path, pos) =
  Loc(pos, path, lines[line - 1])
  where (line, _) = pos

------------------------------------------------------------------------------

fromChars(path, chars) =
  scan(nextPos, (1, 1), chars ++ [EOF])
  |> map(makeLoc(getLines(chars), path))

getLines(chars) =
  chars
  |> split("\n")
  |> toArray

------------------------------------------------------------------------------

colMarker(col) =
  repeat(" ")
  |> take(col - 1)
  |> join("")

------------------------------------------------------------------------------

showLoc(loc) =
  format(
    "(line {}, column {}) in '{}'\n{}\n{}^",
    [line, col, path, lineStr, colMarker(col)]
  ) where {
    (line, col) = pos
    (pos, path, lineStr) = loc
  }
-- see examples above
symbols.ptls
------------------------------------------------------------------------------
-- Language keywords, operators, and symbols, and associated token types
------------------------------------------------------------------------------

keywords = {
  "if": If,
  "then": Then,
  "else": Else,
  "where": Where,
  "with": With,
  "cond": Cond,
  "case": Case,
  "and": And,
  "or": Or,
  "not": Not,
  "in": In,
  "as": As,
  "true": Bool,
  "false": Bool,
  "for": For,
  "when": When,
  "yield": Yield,
  "import": Import,
  "export": Export,
  "throw": Throw,
  "try": Try,
  "catch": Catch,
}

------------------------------------------------------------------------------

opSyms = {
  "+": Add,
  "-": Sub,
  "*": Mul,
  "/": Div,
  "**": Pow,
  "%": Mod,
  "+=": AddAssign,
  "-=": SubAssign,
  "*=": MulAssign,
  "/=": DivAssign,
  "**=": PowAssign,
  "%=": ModAssign,
  "|>": Pipe,
  "=": Assign,
  "==": Equals,
  "!=": NotEq,
  "<": LessThan,
  ">": GreaterThan,
  "<=": LessEq,
  ">=": GreaterEq,
  "=>": Lambda,
  "\$": Dollar,
  "++": Concat,
}

opSymChars = toSet("/+$>!<*-|=%")

------------------------------------------------------------------------------
-- Keep left and right separate to help keep track of
-- when tokenizer is at the start of a new expression
-- Used to disambiguate negation and subtraction

leftSyms = {
  "(": LParen,
  "{": LBracket,
  "[": LArray,
}

rightSyms = {
  ")": RParen,
  "}": RBracket,
  "]": RArray,
}

------------------------------------------------------------------------------

separators = {
  ";": Semicolon,
  ":": Colon,
  ",": Comma,
}
-- see examples above