Examples
A handful of example programs demonstrating various features of Pointless. Some examples (self-hosting tokenizer and stack-based language vm) span multiple source files.
100Doors.ptls
----------------------------------------------------------
-- Solution to the '100 doors' problem
-- http://rosettacode.org/wiki/100_doors
output =
range(1, 100)
|> map(visit(100))
|> println
----------------------------------------------------------
toggle(state) =
if state == Closed then Open else Closed
----------------------------------------------------------
-- Door state on iteration i is recursively
-- defined in terms of previous door state
visit(i, index) = cond {
case (i == 0) Closed
case (index % i == 0) toggle(lastState)
else lastState
} where lastState = visit(i - 1, index)
$ bin/pointless examples/100Doors/100Doors.ptls
[Open, Closed, Closed, Open, Closed, Closed, Closed, Closed, Open, Closed ...]
beer.ptls
-----------------------------------------------------------
-- Print the lyrics to the song '99 bottles of beer'
output =
range(99, 1)
|> map(showBeer)
|> printLines
beerFmt = """{} of beer on the wall!
{} of beer!
You take one down, pass it around
{}"""
showBeer(n) =
format(
beerFmt,
[showBottle(n), showBottle(n), nextBeer(n - 1)]
)
nextBeer(n) =
if n == 0 then "No more bottles of beer on the wall!"
else format("{} of beer on the wall!\n", [showBottle(n)])
-----------------------------------------------------------
-- Get appropriate singular / plural form of 'n bottle(s)'
showBottle(n) =
format("{} {}", [n, bottleStr])
where bottleStr = if n == 1 then "bottle" else "bottles"
$ bin/pointless examples/beer/beer.ptls
...
2 bottles of beer on the wall!
2 bottles of beer!
You take one down, pass it around
1 bottle of beer on the wall!
1 bottle of beer on the wall!
1 bottle of beer!
You take one down, pass it around
No more bottles of beer on the wall!
chart.ptls
-----------------------------------------------------------
-- Given a list of numerical values, print an ascii bar-
-- chart with dimensions (in terms of characters)
-- height x length(values), where the largest value takes
-- up the full height of the chart
-- Negative values get truncated to zero
scale(height, values) =
if values == Empty then ""
else
values
|> normalize(height)
|> getRows(height)
|> join("\n")
-----------------------------------------------------------
-- Get list of scaled values so that max(scaled) = height
normalize(height, values) =
values
|> map(mul(height / maximum(values)))
|> map(max(0))
-----------------------------------------------------------
-- Build up the chart row-by-row starting with the hightest
-- row index (from top to bottom)
getRows(height, values) =
for row in reverse(range(0, height - 1))
yield rowChars(row, values) |> join("")
rowChars(row, values) =
values |> map(getBar(row))
-----------------------------------------------------------
-- BarHeight is the height that the value n projects
-- above the base of the current row (capped at max value
-- seven when the value spans the entire row height)
-- Use this value to index into the array of bar chars
getBar(row, n) = cond {
case barHeight < 0 and row > 0 " "
case barHeight < 0 and row == 0 "_"
else bars[min(7, toInt(barHeight * 7))]
} where {barHeight = (n - row); bars = toArray("▁▂▃▄▅▆▇█")}
-- chart.ptls doesn't have an output variable defined, so we can't execute it directly,
-- but we can import it to use its functions in other scripts, as seen in collatz.ptls
collatz.ptls
import "../chart/chart.ptls" as chart
-----------------------------------------------------------
-- Calculate each number in the collatz sequence for some
-- starting value, and ending when the value 1 is reached
-- Display these values in an ascii bar-chart with 8 rows
output =
iterate(step, 175) -- sequence starts at 175
|> takeUntil(eq(1))
|> chart.scale(4)
|> println
-----------------------------------------------------------
-- get the next number in a collatz sequence
step(n) =
if n % 2 == 0 then n / 2 else n * 3 + 1
$ bin/pointless examples/collatz/collatz.ptls
▂ █
▁ █ ▅ █▁
▃ █▁█▅ ▂ ▆ █▃██▁
▁▂▁▃▂▄▂▆▃▂▅▃▂▁▂▁▃▂▄▂▁▃▂▄▂▁▃▂▅▃▇▄█▅████▆▃█▅█▇█████▄▂▆▃▂▄▂▁▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
echoLines.ptls
-----------------------------------------------------------
-- Read lines from stdin, and print each line along
-- with its line number, right-justified to 3 chars
output =
readLines
|> enumerate
|> map(format("{>3}: {}"))
|> printLines
$ cat examples/echoLines/langs.txt | bin/pointless examples/echoLines/echoLines.ptls
0: c++:
1: "The position you applied for is no longer available"
2:
3: java:
4: "We sincerely appreciated your enthusiasm, outstanding
5: interview, impeccible resume, and engaging personality.
6: It is rare to find a candidate as qualified and capable
7: as yourself. That being said, we regret to inform you that..."
8:
9: python:
10: "... and we provide each of our software engineers with a standing desk",
11: "Actually, I'd prefer a normal desk if that's...",
12: "WE PROVIDE EACH OF OUR SOFTWARE ENGINEERS WITH A STANDING DESK"
13:
14: haskell:
15: "t o = whomSt :: so|ever| <*> it m $ ay conc . ern; it
16: | is ++ m y !Pleasur (*e) ~~>> to >>= _ \i -> n fo (R m y o !! u)
17: | that -> [yo..u"
18:
19: javascript:
20: "click here to learn all of ES6 in 20 seconds!
21: http://obliterating.the.coding.interview.io/"
render.ptls
fen = "1Q6/5pk1/2p3p1/1p2N2p/1b5P/1bn5/2r3P1/2K5"
output =
fen
|> showBoard
|> renderHTML
|> println
------------------------------------------------------------------------------
showBoard(fen) =
fen
|> split("/")
|> map(expandRow)
|> join("\n")
------------------------------------------------------------------------------
expandRow(row) =
row
|> toList
|> map(expandChar)
|> join("")
expandChar(char) =
getDefault(spaceDict, char, char)
spaceDict = {
"8": " ",
"7": " ",
"6": " ",
"5": " ",
"4": " ",
"3": " ",
"2": " ",
"1": " ",
}
------------------------------------------------------------------------------
colors =
[Light, Dark] |> repeat |> concat
renderHTML(boardStr) =
boardStr
|> toList
|> zip(colors)
|> map(getIndex(renderSyms))
|> join("")
|> formatTemplate
------------------------------------------------------------------------------
renderSyms = {
(Dark, "R" ): "",
(Dark, "r" ): "",
(Dark, "N" ): "",
(Dark, "n" ): "",
(Dark, "B" ): "",
(Dark, "b" ): "",
(Dark, "Q" ): "",
(Dark, "q" ): "",
(Dark, "K" ): "",
(Dark, "k" ): "",
(Dark, "P" ): "",
(Dark, "p" ): "",
(Dark, " " ): "",
(Dark, "\n"): "\n",
(Light, "R" ): "♖",
(Light, "r" ): "♜",
(Light, "N" ): "♘",
(Light, "n" ): "♞",
(Light, "B" ): "♗",
(Light, "b" ): "♝",
(Light, "Q" ): "♕",
(Light, "q" ): "♛",
(Light, "K" ): "♔",
(Light, "k" ): "♚",
(Light, "P" ): "♙",
(Light, "p" ): "♟︎",
(Light, " " ): " ",
(Light, "\n"): "\n",
}
------------------------------------------------------------------------------
styles = """<style>
@font-face {
font-family: "merida";
src: url("merida.woff2") format("woff2");
}
.board {
font-family: "merida";
font-size: 40px;
border: 1px solid #ccc;
display: inline-block;
padding: 3px;
}
</style>
"""
formatTemplate(boardChars) =
format("{}\n<pre class='board'>{}</pre>", [styles, boardChars])
Rendered result:
♟︎
♟︎ ♟︎
♟︎ ♟︎
♝
♜ ♙
guess.ptls
number = randRange(0, 1000)
getResponse(n) =
format("{} is {}", [n, compareGuess(n)])
compareGuess(n) = cond {
case n > number "too high"
case n < number "too low"
case n == number "correct!"
}
----------------------------------------------------------
-- A number guessing game (binary search)
-- User enters an integer guess between 0 and 1000
-- progarm reports whether guess is above or below the
-- target number - continues until correct guess is entered
output =
readLines
|> map(toInt)
|> takeUntil(eq(number))
|> map(getResponse)
|> printLines
$ bin/pointless examples/guess/guess.ptls
500
500 is too high
250
250 is too low
375
375 is too high
312
312 is correct!
life.ptls
-----------------------------------------------------------
-- Print 100 simulated states of conway's game of life
-- for a glider starting pattern
-- Print generation number along with cells
output =
initCells
|> iterate(updateCells)
|> take(20)
|> enumerate
|> map(showPair)
|> printFrames
-- need to concat results from printFrames to join the
-- commands into one big sequence
initCells =
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
width = length(initCells[0])
height = length(initCells)
positions =
for y in range(0, height - 1)
for x in range(0, width - 1)
yield (x, y)
-----------------------------------------------------------
-- Update each cell in the grid according to its position,
-- and convert the resulting list back to a 2D array
updateCells(cells) =
positions
|> map(tick(cells))
|> toNDArray((height, width))
-----------------------------------------------------------
-- Get the new value for a cell at a give position
-- based on the current cell values in the grid
tick(cells, pos) = toInt(survive or birth) where {
survive = cells[y][x] == 1 and count in {2, 3}
birth = cells[y][x] == 0 and count == 3
count = getCount(x, y, cells)
(x, y) = pos
}
-----------------------------------------------------------
-- Get the number of live neighbors of a given position
getCount(x, y, cells) = sum(neighborVals) where {
neighborVals =
for dx in deltas
for dy in deltas
when (dx != 0 or dy != 0)
yield getNeighbor(x + dx, y + dy, cells)
deltas = [-1, 0, 1]
}
getNeighbor(x, y, cells) = cells[y % height][x % width]
-----------------------------------------------------------
-- Print the board and generation number given pairs
-- of (gen, cells) from the enumerate function
showPair(pair) =
format("{}\ngeneration: {}", [showCells(cells), gen])
where (gen, cells) = pair
showCells(cells) =
toList(cells)
|> map(showRow)
|> join("\n")
showRow(row) =
format("|{}|", [map(showCell, toList(row)) |> join("")])
showCell(cell) =
if cell == 1 then "*" else " "
$ bin/pointless examples/life/life.ptls
| |
| * |
| * |
| *** |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
generation: 0
| |
| |
| * * |
| ** |
| * |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
generation: 1
| |
| |
| * |
| * * |
| ** |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
generation: 2
...
permutations.ptls
-----------------------------------------------------------
-- For a list of values, get a list of all permutations of
-- the items in the original list
-- (algorithm produces duplicate permutations for lists
-- containing duplicate elements)
permutations(list) = cond {
case isEmpty(list) []
case isEmpty(tail(list)) [list]
else
for elem in list
for perm in permutations(delete(elem, list))
yield [elem] ++ perm
}
-----------------------------------------------------------
delete(elem, list) = cond {
case isEmpty(list) []
case (head(list) == elem) tail(list)
else [head(list)] ++ delete(elem, tail(list))
}
-----------------------------------------------------------
output =
range(0, 2) -- this example uses the list [0, 1, 2]
|> permutations
|> println
$ bin/pointless examples/permutations/permutations.ptls
[[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]]
quine.ptls
output = (s => println(s + show(toTuple([s]))))("output = (s => println(s + show(toTuple([s]))))")
$ bin/pointless examples/quine/quine.ptls
output = (s => println(s + show(toTuple([s]))))("output = (s => println(s + show(toTuple([s]))))")
factorsVM.ptls
import "stackVM.ptls" as stackVM
------------------------------------------------------------------------------
output =
program
|> stackVM.runGetOutput
|> printLines
------------------------------------------------------------------------------
n = 80122
program = [
Const(2),
Store(0), -- store value 2 in slot 0 (initial divisor)
Const(n),
Store(1), -- store value n in slot 1 (initial dividend)
Label("checkDone"),
Load(1), -- load dividend
Const(1),
Eq, -- compare dividend to value 1
JmpIf("end"), -- jump to end if dividend == 1
Label("checkDiv"),
Load(1), -- load dividend
Load(0), -- load divisor
Mod,
Const(0),
Eq, -- compare dividend % dividor == 0
JmpIf("divisible"),
Load(0), -- load divisor
Const(1),
Add, -- increment divisor
Store(0), -- store new divisor
Jmp("checkDiv"), -- check end condition
Label("divisible"),
Load(0), -- load divisor
Print, -- print divisor (current factor)
Load(1), -- load dividend
Load(0), -- load divisor
Div,
Store(1), -- store dividend / divisor as new dividend
Jmp("checkDone"), -- check end condition
Label("end"),
Const("done"),
Print, -- print "done"
Exit,
]
$ bin/pointless examples/stackVM/factorsVM.ptls
2
7
59
97
done
collatzVM.ptls
import "stackVM.ptls" as stackVM
------------------------------------------------------------------------------
output =
program
|> stackVM.runGetOutput
|> printLines
------------------------------------------------------------------------------
n = 870
program = [
Const(n),
Label("startLoop"),
Store(0),
Load(0),
Const(1),
Eq,
JmpIf("end"),
Load(0),
Print,
Load(0),
Const(2),
Mod,
Const(0),
Eq,
JmpIf("even"),
Load(0),
Const(3),
Mul,
Const(1),
Add,
Jmp("startLoop"),
Label("even"),
Load(0),
Const(2),
Div,
Jmp("startLoop"),
Label("end"),
Load(0),
Print,
Const("done"),
Print,
Exit,
]
$ bin/pointless examples/stackVM/collatzVM.ptls
870
435.0
...
4.0
2.0
1.0
done
stackVM.ptls
export {
runGetOutput
}
------------------------------------------------------------------------------
import "instructions.ptls" as instructions
------------------------------------------------------------------------------
vmFromInsts(program) = VM {
slots = zeroArray(8)
insts = toArray(instructions.convertJumps(program))
index = 0
stack = []
outVal = None
}
------------------------------------------------------------------------------
runGetOutput(insts) =
vmFromInsts(insts)
|> iterate(compose(setOutput(None), eval))
|> takeUntil(vm => vm.insts[vm.index] == Exit)
|> map(vm => vm.outVal)
|> filter(notEq(None))
------------------------------------------------------------------------------
eval(vm) = cond {
case is(Const, inst)
vm
|> pushVal(arg)
|> advance
case is(Print, inst)
vm
|> popVals(1)
|> setOutput(valAt(0, vm))
|> advance
case is(Load, inst)
vm
|> load(arg)
|> advance
case is(Store, inst)
vm
|> store(arg)
|> popVals(1)
|> advance
case is(JmpIf, inst)
vm
|> popVals(1)
|> jumpIf(valAt(0, vm), arg)
case is(Jmp, inst)
vm
|> jumpIf(true, arg)
case is(Eq, inst) vm |> binaryOp(eq)
case is(Add, inst) vm |> binaryOp(add)
case is(Sub, inst) vm |> binaryOp(sub)
case is(Mul, inst) vm |> binaryOp(mul)
case is(Div, inst) vm |> binaryOp(div)
case is(Mod, inst) vm |> binaryOp(mod)
} where {
inst = vm.insts[vm.index]
arg = unwrap(inst)
}
------------------------------------------------------------------------------
advance(vm) = vm with $.index += 1
pushVal(arg, vm) = vm with $.stack = [arg] ++ vm.stack
popVals(n, vm) = vm with $.stack = drop(n, vm.stack)
setOutput(val, vm) = vm with $.outVal = val
store(arg, vm) = vm with $.slots[arg] = head(vm.stack)
load(arg, vm) = pushVal(vm.slots[arg], vm)
jumpIf(pred, arg, vm) =
if pred
then vm with $.index = arg
else advance(vm)
valAt(n, vm) = at(n, vm.stack)
binaryOp(op, vm) =
vm
|> popVals(2)
|> pushVal(op(valAt(0, vm), valAt(1, vm)))
|> advance
-- see examples above
instructions.ptls
export {
showInst, makeInstArray
}
------------------------------------------------------------------------------
showInst(inst) =
format("[ {<5} ] {}", [getLabel(inst), argOrBlank(inst)])
argOrBlank(inst) =
if is(PtlsTuple, inst) then unwrap(inst) else ""
------------------------------------------------------------------------------
convertJumps(insts) =
insts
|> filter(notIs(Label))
|> map(convertJump(inds))
where inds = getLabelInds(insts)
------------------------------------------------------------------------------
convertJump(inds, inst) = cond {
case is(Jmp, inst) Jmp(inds[unwrap(inst)])
case is(JmpIf, inst) JmpIf(inds[unwrap(inst)])
else inst
}
------------------------------------------------------------------------------
getLabelInds(insts) =
insts
|> reduce(scanInst, (0, {})) -- keep track of (currentIndex, indexMap)
|> at(1) -- return index map
------------------------------------------------------------------------------
scanInst(pair, inst) =
if not is(Label, inst)
then (ind + 1, inds)
else (ind, newInds)
where {
newInds = inds with $[unwrap(inst)] = ind
(ind, inds) = pair
}
-- see examples above
tokenize.ptls
import "location.ptls" as location
import "symbols.ptls" as symbols
------------------------------------------------------------------------------
-- For tokenizer tok with chars c0, c1, c2 ... cn
-- and funcs f0, f1, f2, ... fn
-- return true if n >= m and all([f0(c0), f1(c1), ...])
matchChars(funcs, tok) =
if length(pairs) < length(funcs) then false
else
pairs
|> map(pair => (func(char) where (func, char) = pair))
|> all
where pairs = zip(funcs, tok.chars)
------------------------------------------------------------------------------
-- New tokenizer object
new(text) = Tokenizer {
chars = split("", text)
locs = location.fromChars("path", chars)
stack = []
tokens = []
tokenLoc = head(locs) -- where the current token started
}
------------------------------------------------------------------------------
-- Move forward one char, push char onto stack
advance(tok) =
tok with {
$.chars = tail(tok.chars)
$.stack = [head(tok.chars)] ++ tok.stack
$.locs = tail(tok.locs)
}
------------------------------------------------------------------------------
-- Skip chars (don't push to stack)
skipVal(tok) = advance(tok) with {
-- keep stack of tok before advance (leaves out current char)
$.stack = tok.stack
}
------------------------------------------------------------------------------
-- Push chars to stack as long as matchChars(funcs)
advanceWhile(funcs, tok) =
iterate(advance, tok)
|> dropWhile(matchChars(funcs))
|> head
------------------------------------------------------------------------------
-- Skip chars (don't push to stack) as long as matchChars(funcs)
skipWhile(funcs, tok) =
iterate(skipVal, tok)
|> dropWhile(matchChars(funcs))
|> head
------------------------------------------------------------------------------
-- No more chars to parse
isEOF(tok) = tok.chars == Empty
------------------------------------------------------------------------------
-- Get token string from stack of tokens
tokVal(tok) =
tok.stack
|> reverse -- pushing chars reversed order - reverse again to correct
|> join("")
------------------------------------------------------------------------------
-- Push new token to token stack
makeToken(labelFunc, tok) = tok with {
$.tokens = [token] ++ tok.tokens
$.stack = [] -- clear stack
$.tokenLoc = head(tok.locs) -- set loc for next token
} where {
-- add label to token object (Name {...})
token = wrapObject(labelFunc(val), {value = val; loc = tok.tokenLoc})
val = tokVal(tok)
}
------------------------------------------------------------------------------
-- Find matching token type for upcoming chars
isBlank = matchChars([eq("_")])
isComment = matchChars([eq("-"), eq("-")])
isInt = matchChars([inFunc(digits)])
isFloat = matchChars([eq("."), inFunc(digits)])
isName = matchChars([inFunc(lowers)])
isLabel = matchChars([inFunc(uppers)])
isCustomField = matchChars([eq("."), inFunc(alphas)])
isLangField = matchChars([eq("."), eq("!"), inFunc(alphas)])
isOpSym = matchChars([inFunc(symbols.opSymChars)])
isWhitespace = matchChars([eq(" ")])
isNewline = matchChars([eq("\n")])
isSeparator = matchChars([inFunc(symbols.separators)])
isLeftSym = matchChars([inFunc(symbols.leftSyms)])
isRightSym = matchChars([inFunc(symbols.rightSyms)])
isString = matchChars([eq("\"")])
------------------------------------------------------------------------------
-- Make a new token and push to token stack
-- must have not isEOF(tok)
-- throws TokenizerError
getToken(tok) = cond {
case isBlank(tok)
tok
|> skipVal -- skip blank char
|> makeToken(const(Blank))
----------------------------------------------------------------------------
case isComment(tok)
tok
|> advanceWhile([eq("-")]) -- take all dashes
|> advanceWhile([notEq("\n")]) -- comment extends to the end of line
|> makeToken(const(Comment))
----------------------------------------------------------------------------
case isWhitespace(tok)
tok
|> skipWhile([eq(" ")]) -- combine repeated spaces into one token
|> makeToken(const(Whitespace))
----------------------------------------------------------------------------
case isNewline(tok)
tok
|> skipWhile([eq("\n")]) -- combine repeated newlines into one token
|> makeToken(const(Newline))
----------------------------------------------------------------------------
case isSeparator(tok)
tok
|> advance -- take separator char
|> makeToken(getIndex(symbols.separators))
----------------------------------------------------------------------------
case isName(tok)
tok
|> advanceWhile([inFunc(alNums)]) -- take all alphanumeric chars
-- return matching keyword token if name is a keyword
|> makeToken(getDefault(symbols.keywords, Name))
----------------------------------------------------------------------------
case isCustomField(tok)
tok
|> skipVal -- take '.'
|> advanceWhile([inFunc(alNums)])
|> makeToken(const(Field))
----------------------------------------------------------------------------
case isLangField(tok)
tok
|> skipVal -- take '.'
|> advance -- take '!'
|> advanceWhile([inFunc(alNums)])
|> makeToken(const(Field))
----------------------------------------------------------------------------
case isLabel(tok)
tok
|> advanceWhile([inFunc(alNums)]) -- take all alphanumeric chars
|> makeToken(const(Label))
----------------------------------------------------------------------------
case isString(tok)
tok
|> skipVal -- skip opening '"'
|> getStringChars
|> skipVal -- skip closing '"'
|> makeToken(const(String))
----------------------------------------------------------------------------
case isInt(tok)
tok
|> advanceWhile([inFunc(digits)]) -- take all leading digits
-- if there's a decimal point followed by more digits, take those too
|> (tok => if isFloat(tok) then getFloatDigits(tok) else tok)
|> makeToken(const(Number))
----------------------------------------------------------------------------
case isFloat(tok)
tok
|> getFloatDigits(tok) -- get decimal point and following digits
|> makeToken(const(Number))
----------------------------------------------------------------------------
case isOpSym(tok)
tok
|> advanceWhile([inFunc(symbols.opSymChars)]) -- take all operator chars
|> checkOpSym -- check that chars form a valid operator
|> makeToken(getIndex(symbols.opSyms))
----------------------------------------------------------------------------
-- leftSyms and rightSyms kept separate to accomodate Sub -> Neg conversion
case isLeftSym(tok)
tok
|> advance -- take leftSym char
|> makeToken(getIndex(symbols.leftSyms))
----------------------------------------------------------------------------
case isRightSym(tok)
tok
|> advance -- take rightSym char
|> makeToken(getIndex(symbols.rightSyms))
----------------------------------------------------------------------------
-- leading chars don't form a valid token
-- example: a non alphaNumeric char not in any operator or symbol like '&'
else
tokError(tok, message)
where message = format("Unexpected symbol '{}'", [head(tok.chars)])
} requires not isEOF(tok)
------------------------------------------------------------------------------
-- Get chars inside string
getStringChars(tok) = cond {
case isEOF(tok)
-- reached end of file before closing quote
tokError(tok, "Unmatched quote")
case matchChars([eq("\n")], tok)
-- reached end of line before closing quote
tokError(tok, "Unmatched quote (must escape line breaks in string)")
case matchChars([eq("\\")], tok)
-- on backslash, move past backslash and escape char
getStringChars(advance(advance(tok)))
case matchChars([eq("\"")], tok)
tok -- stop before taking closing quote
else getStringChars(advance(tok))
}
------------------------------------------------------------------------------
-- Get decimal point and following digits
getFloatDigits(tok) =
tok
|> advance
|> advanceWhile([inFunc(digits)])
------------------------------------------------------------------------------
-- Check that current token chars form valid operator
checkOpSym(tok) =
if val in symbols.opSyms then tok
else tokError(tok, format("Invalid operator '{}'", [val]))
where val = tokVal(tok)
------------------------------------------------------------------------------
-- Throw TokenizerError
tokError(tok, message) = throw TokenizerError(fullMessage)
where fullMessage = format(
"{}\nTokenizer Error:\n\n{}\n{}\n\n{}",
[sep, message, sep, location.showLoc(tok.tokenLoc)]
)
sep = repeat("-") |> take(79) |> join("")
------------------------------------------------------------------------------
-- Get string representation for token
showTok(token) =
format(
"{>3}:{<2} [ {<12} ] {}",
[line, col, getLabel(token), token.value]
) where (line, col) = at(0, token.loc)
------------------------------------------------------------------------------
startToks = toSet(
vals(symbols.opSyms)
++ vals(symbols.separators)
++ vals(symbols.leftSyms)
++ vals(symbols.keywords)
)
endToks = toSet(
vals(symbols.rightSyms)
++ [Name, Field, String, Number]
)
updateIsStart(isStart, token) = cond {
case getLabel(token) in startToks true
case getLabel(token) in endToks false
else isStart
}
------------------------------------------------------------------------------
getTokens(tok) =
tok
|> iterate(getToken)
|> drop(1)
|> takeUntil(isEOF)
|> map(tok => head(tok.tokens))
------------------------------------------------------------------------------
convert(tokens) =
scan(updateIsStart, false, tokens)
|> zip(tokens)
|> map(convertSubExpr)
|> getNeighborTruples
|> map(convertSubPadding)
------------------------------------------------------------------------------
-- Scan tokens, keeping track of whether '-' corresponds to a negative
-- sign or subtraction operator, depending on whether they occur at the start
-- of an expression, and update (all '-' are initially parsed as subtraction)
convertSubExpr(pair) =
if is(Sub, token) and isStart
then wrapObject(Neg, token)
else token
where (token, isStart) = pair
------------------------------------------------------------------------------
getNeighborTruples(tokens) =
zipN([tokens, [None] ++ tokens, drop(1, tokens)])
------------------------------------------------------------------------------
-- Special case: conver '-' to neg in 'a -b'
-- for examplem:
-- array = [1 2 -3]
convertSubPadding(triple) =
if is(Sub, current) and is(Whitespace, last) and is(Number, next)
then wrapObject(Neg, current)
else current
where (current, last, next) = triple
------------------------------------------------------------------------------
text = readLines |> join("\n")
output =
try
new(text)
|> getTokens
|> convert
|> map(showTok)
|> eager
|> printLines
catch is(TokenizerError)
err => println(unwrap(err))
$ cat examples/tokenizer/tokenizer.ptls | bin/pointless examples/tokenizer/tokenizer.ptls
1:1 [ Newline ]
2:1 [ Import ] import
2:7 [ Whitespace ]
2:8 [ String ] location.ptls
2:23 [ Whitespace ]
2:24 [ As ] as
2:26 [ Whitespace ]
2:27 [ Name ] location
2:35 [ Newline ]
3:1 [ Import ] import
3:7 [ Whitespace ]
3:8 [ String ] symbols.ptls
3:22 [ Whitespace ]
3:23 [ As ] as
...
location.ptls
export {fromChars, showLoc}
------------------------------------------------------------------------------
nextPos(pos, char) =
if char == "\n" then (line + 1, 1) else (line, col + 1)
where (line, col) = pos
------------------------------------------------------------------------------
makeLoc(lines, path, pos) =
Loc(pos, path, lines[line - 1])
where (line, _) = pos
------------------------------------------------------------------------------
fromChars(path, chars) =
scan(nextPos, (1, 1), chars ++ [EOF])
|> map(makeLoc(getLines(chars), path))
getLines(chars) =
chars
|> split("\n")
|> toArray
------------------------------------------------------------------------------
colMarker(col) =
repeat(" ")
|> take(col - 1)
|> join("")
------------------------------------------------------------------------------
showLoc(loc) =
format(
"(line {}, column {}) in '{}'\n{}\n{}^",
[line, col, path, lineStr, colMarker(col)]
) where {
(line, col) = pos
(pos, path, lineStr) = loc
}
-- see examples above
symbols.ptls
------------------------------------------------------------------------------
-- Language keywords, operators, and symbols, and associated token types
------------------------------------------------------------------------------
keywords = {
"if": If,
"then": Then,
"else": Else,
"where": Where,
"with": With,
"cond": Cond,
"case": Case,
"and": And,
"or": Or,
"not": Not,
"in": In,
"as": As,
"true": Bool,
"false": Bool,
"for": For,
"when": When,
"yield": Yield,
"import": Import,
"export": Export,
"throw": Throw,
"try": Try,
"catch": Catch,
}
------------------------------------------------------------------------------
opSyms = {
"+": Add,
"-": Sub,
"*": Mul,
"/": Div,
"**": Pow,
"%": Mod,
"+=": AddAssign,
"-=": SubAssign,
"*=": MulAssign,
"/=": DivAssign,
"**=": PowAssign,
"%=": ModAssign,
"|>": Pipe,
"=": Assign,
"==": Equals,
"!=": NotEq,
"<": LessThan,
">": GreaterThan,
"<=": LessEq,
">=": GreaterEq,
"=>": Lambda,
"\$": Dollar,
"++": Concat,
}
opSymChars = toSet("/+$>!<*-|=%")
------------------------------------------------------------------------------
-- Keep left and right separate to help keep track of
-- when tokenizer is at the start of a new expression
-- Used to disambiguate negation and subtraction
leftSyms = {
"(": LParen,
"{": LBracket,
"[": LArray,
}
rightSyms = {
")": RParen,
"}": RBracket,
"]": RArray,
}
------------------------------------------------------------------------------
separators = {
";": Semicolon,
":": Colon,
",": Comma,
}
-- see examples above
