Files
block-game/ServerScriptService/Actor/ServerChunkManager/TerrainGen/Deflate/Huffman/BitBuffer.lua
2026-01-08 22:58:58 +02:00

1816 lines
66 KiB
Lua

local CHAR_SET = [[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/]]
-- Tradition is to use chars for the lookup table instead of codepoints.
-- But due to how we're running the encode function, it's faster to use codepoints.
local encode_char_set = {}
local decode_char_set = {}
for i = 1, 64 do
encode_char_set[i - 1] = string.byte(CHAR_SET, i, i)
decode_char_set[string.byte(CHAR_SET, i, i)] = i - 1
end
-- stylua: ignore
local HEX_TO_BIN = {
["0"] = "0000", ["1"] = "0001", ["2"] = "0010", ["3"] = "0011",
["4"] = "0100", ["5"] = "0101", ["6"] = "0110", ["7"] = "0111",
["8"] = "1000", ["9"] = "1001", ["a"] = "1010", ["b"] = "1011",
["c"] = "1100", ["d"] = "1101", ["e"] = "1110", ["f"] = "1111"
}
-- stylua: ignore
local NORMAL_ID_VECTORS = { -- [Enum.Value] = Vector3.fromNormalId(Enum)
[0] = Vector3.new(1, 0, 0), -- Enum.NormalId.Right
[1] = Vector3.new(0, 1, 0), -- Enum.NormalId.Top
[2] = Vector3.new(0, 0, 1), -- Enum.NormalId.Back
[3] = Vector3.new(-1, 0, 0), -- Enum.NormalId.Left
[4] = Vector3.new(0, -1, 0), -- Enum.NormalId.Bottom
[5] = Vector3.new(0, 0, -1) -- Enum.NormalId.Front
}
local ONES_VECTOR = Vector3.new(1, 1, 1)
local BOOL_TO_BIT = { [true] = 1, [false] = 0 }
local CRC32_POLYNOMIAL = 0xedb88320
local crc32_poly_lookup = {}
for i = 0, 255 do
local crc = i
for _ = 1, 8 do
local mask = -bit32.band(crc, 1)
crc = bit32.bxor(bit32.rshift(crc, 1), bit32.band(CRC32_POLYNOMIAL, mask))
end
crc32_poly_lookup[i] = crc
end
local powers_of_2 = {}
for i = 0, 64 do
powers_of_2[i] = 2 ^ i
end
local byte_to_hex = {}
for i = 0, 255 do
byte_to_hex[i] = string.format("%02x", i)
end
local function bitBuffer(stream)
if stream ~= nil then
assert(type(stream) == "string", "argument to BitBuffer constructor must be either nil or a string")
end
-- The bit buffer works by keeping an array of bytes, a 'final' byte, and how many bits are currently in that last byte
-- Bits are not kept track of on their own, and are instead combined to form a byte, which is stored in the last space in the array.
-- This byte is also stored seperately, so that table operations aren't needed to read or modify its value.
-- The byte array is called `bytes`. The last byte is stored in `lastByte`. The bit counter is stored in `bits`.
local bits = 0 -- How many free floating bits there are.
local bytes = {} --! -- Array of bytes currently in the buffer
local lastByte = 0 -- The most recent byte in the buffer, made up of free floating bits
local byteCount = 0 -- This variable keeps track of how many bytes there are total in the bit buffer.
local bitCount = 0 -- This variable keeps track of how many bits there are total in the bit buffer
local pointer = 0 -- This variable keeps track of what bit the read functions start at
local pointerByte = 1 -- This variable keeps track of what byte the pointer is at. It starts at 1 since the byte array starts at 1.
if stream then
byteCount = #stream
bitCount = byteCount * 8
bytes = table.create(#stream)
for i = 1, byteCount do
bytes[i] = string.byte(stream, i, i)
end
end
local function dumpBinary()
-- This function is for debugging or analysis purposes.
-- It dumps the contents of the byte array and the remaining bits into a string of binary digits.
-- Thus, bytes [97, 101] with bits [1, 1, 0] would output "01100001 01100101 110"
local output = table.create(byteCount) --!
for i, v in ipairs(bytes) do
output[i] = string.gsub(byte_to_hex[v], "%x", HEX_TO_BIN)
end
if bits ~= 0 then
-- Because the last byte (where the free floating bits are stored) is in the byte array, it has to be overwritten.
output[byteCount] = string.sub(output[byteCount], 1, bits)
end
return table.concat(output, " ")
end
local function dumpStringOld()
-- This function is for accessing the total contents of the bitbuffer.
-- This function combines all the bytes, including the last byte, into a string of binary data.
-- Thus, bytes [97, 101] and bits [1, 1, 0] would become (in hex) "0x61 0x65 0x06"
-- It's substantially faster to create several smaller strings before using table.concat. (well maybe it was, but it isn't now post 2022)
local output = table.create(math.ceil(byteCount / 4096)) --!
local c = 1
for i = 1, byteCount, 4096 do -- groups of 4096 bytes is the point at which there are diminishing returns
output[c] = string.char(table.unpack(bytes, i, math.min(byteCount, i + 4095)))
c = c + 1
end
return table.concat(output, "")
end
--Let lua be lua
local function dumpString()
return string.char(table.unpack(bytes))
end
local function dumpHex()
-- This function is for getting the hex of the bitbuffer's contents, should that be desired
local output = table.create(byteCount) --!
for i, v in ipairs(bytes) do
output[i] = byte_to_hex[v]
end
return table.concat(output, "")
end
local function dumpBase64()
-- Base64 is a safe and easy way to convert binary data to be entirely printable
-- It works on the principle that groups of 3 bytes (24 bits) can evenly be divided into 4 groups of 6
-- And 2^6 is a mere 64, far less than the number of printable characters.
-- If there are any missing bytes, `=` is added to the end as padding.
-- Base64 increases the size of its input by 33%.
local output = table.create(math.ceil(byteCount * 1.333)) --!
local c = 1
for i = 1, byteCount, 3 do
local b1, b2, b3 = bytes[i], bytes[i + 1], bytes[i + 2]
local packed = bit32.bor(bit32.lshift(b1, 16), bit32.lshift(b2 or 0, 8), b3 or 0)
-- This can be done with bit32.extract (and/or bit32.lshift, bit32.band, bit32.rshift)
-- But bit masking and shifting is more eloquent in my opinion.
output[c] = encode_char_set[bit32.rshift(bit32.band(packed, 0xfc0000), 0x12)]
output[c + 1] = encode_char_set[bit32.rshift(bit32.band(packed, 0x3f000), 0xc)]
output[c + 2] = b2 and encode_char_set[bit32.rshift(bit32.band(packed, 0xfc0), 0x6)] or 0x3d -- 0x3d == "="
output[c + 3] = b3 and encode_char_set[bit32.band(packed, 0x3f)] or 0x3d
c = c + 4
end
c = c - 1 -- c will always be 1 more than the length of `output`
local realOutput = table.create(math.ceil(c / 0x1000)) --!
local k = 1
for i = 1, c, 0x1000 do
realOutput[k] = string.char(table.unpack(output, i, math.min(c, i + 0xfff)))
k = k + 1
end
return table.concat(realOutput, "")
end
local function exportChunk(chunkLength)
assert(type(chunkLength) == "number", "argument #1 to BitBuffer.exportChunk should be a number")
assert(chunkLength > 0, "argument #1 to BitBuffer.exportChunk should be above zero")
assert(chunkLength % 1 == 0, "argument #1 to BitBuffer.exportChunk should be an integer")
-- Since `i` is being returned, the most eloquent way to handle this is with a coroutine
-- This allows returning the existing value of `i` without having to increment it first.
-- The alternative was starting at `i = -(chunkLength-1)` and incrementing at the start of the iterator function.
return coroutine.wrap(function()
local realChunkLength = chunkLength - 1
-- Since this function only has one 'state', it's perfectly fine to use a for-loop.
for i = 1, byteCount, chunkLength do
local chunk = string.char(table.unpack(bytes, i, math.min(byteCount, i + realChunkLength)))
coroutine.yield(i, chunk)
end
end)
end
local function exportBase64Chunk(chunkLength)
chunkLength = chunkLength or 76
assert(type(chunkLength) == "number", "argument #1 to BitBuffer.exportBase64Chunk should be a number")
assert(chunkLength > 0, "argument #1 to BitBuffer.exportBase64Chunk should be above zero")
assert(chunkLength % 1 == 0, "argument #1 to BitBuffer.exportBase64Chunk should be an integer")
local output = table.create(math.ceil(byteCount * 0.333)) --!
local c = 1
for i = 1, byteCount, 3 do
local b1, b2, b3 = bytes[i], bytes[i + 1], bytes[i + 2]
local packed = bit32.bor(bit32.lshift(b1, 16), bit32.lshift(b2 or 0, 8), b3 or 0)
output[c] = encode_char_set[bit32.rshift(bit32.band(packed, 0xfc0000), 0x12)]
output[c + 1] = encode_char_set[bit32.rshift(bit32.band(packed, 0x3f000), 0xc)]
output[c + 2] = b2 and encode_char_set[bit32.rshift(bit32.band(packed, 0xfc0), 0x6)] or 0x3d
output[c + 3] = b3 and encode_char_set[bit32.band(packed, 0x3f)] or 0x3d
c = c + 4
end
c = c - 1
return coroutine.wrap(function()
local realChunkLength = chunkLength - 1
for i = 1, c, chunkLength do
local chunk = string.char(table.unpack(output, i, math.min(c, i + realChunkLength)))
coroutine.yield(chunk)
end
end)
end
local function exportHexChunk(chunkLength)
assert(type(chunkLength) == "number", "argument #1 to BitBuffer.exportHexChunk should be a number")
assert(chunkLength > 0, "argument #1 to BitBuffer.exportHexChunk should be above zero")
assert(chunkLength % 1 == 0, "argument #1 to BitBuffer.exportHexChunk should be an integer")
local halfLength = math.floor(chunkLength / 2)
if chunkLength % 2 == 0 then
return coroutine.wrap(function()
local output = {} --!
for i = 1, byteCount, halfLength do
for c = 0, halfLength - 1 do
output[c] = byte_to_hex[bytes[i + c]]
end
coroutine.yield(table.concat(output, "", 0))
end
end)
else
return coroutine.wrap(function()
local output = { [0] = "" } --!
local remainder = ""
local i = 1
while i <= byteCount do
if remainder == "" then
output[0] = ""
for c = 0, halfLength - 1 do
output[c + 1] = byte_to_hex[bytes[i + c]]
end
local endByte = byte_to_hex[bytes[i + halfLength]]
if endByte then
output[halfLength + 1] = string.sub(endByte, 1, 1)
remainder = string.sub(endByte, 2, 2)
end
i = i + 1
else
output[0] = remainder
for c = 0, halfLength - 1 do
output[c + 1] = byte_to_hex[bytes[i + c]]
end
output[halfLength + 1] = ""
remainder = ""
end
coroutine.yield(table.concat(output, "", 0))
i = i + halfLength
end
end)
end
end
local function crc32()
local crc = 0xffffffff -- 2^32
for _, v in ipairs(bytes) do
local poly = crc32_poly_lookup[bit32.band(bit32.bxor(crc, v), 255)]
crc = bit32.bxor(bit32.rshift(crc, 8), poly)
end
return bit32.bnot(crc) % 0xffffffff -- 2^32
end
local function getLength()
return bitCount
end
local function getByteLength()
return byteCount
end
local function getPointer()
-- This function gets the value of the pointer. This is self-explanatory.
return pointer
end
local function setPointer(n)
assert(type(n) == "number", "argument #1 to BitBuffer.setPointer should be a number")
assert(n >= 0, "argument #1 to BitBuffer.setPointer should be zero or higher")
assert(n % 1 == 0, "argument #1 to BitBuffer.setPointer should be an integer")
assert(n <= bitCount, "argument #1 to BitBuffer.setPointerByte should within range of the buffer")
-- This function sets the value of pointer. This is self-explanatory.
pointer = n
pointerByte = math.floor(n / 8) + 1
end
local function setPointerFromEnd(n)
assert(type(n) == "number", "argument #1 to BitBuffer.setPointerFromEnd should be a number")
assert(n >= 0, "argument #1 to BitBuffer.setPointerFromEnd should be zero or higher")
assert(n % 1 == 0, "argument #1 to BitBuffer.setPointerFromEnd should be an integer")
assert(n <= bitCount, "argument #1 to BitBuffer.setPointerFromEnd should within range of the buffer")
pointer = bitCount - n
pointerByte = math.floor(pointer / 8 + 1)
end
local function getPointerByte()
return pointerByte
end
local function setPointerByte(n)
assert(type(n) == "number", "argument #1 to BitBuffer.setPointerByte should be a number")
assert(n > 0, "argument #1 to BitBuffer.setPointerByte should be positive")
assert(n % 1 == 0, "argument #1 to BitBuffer.setPointerByte should be an integer")
assert(n <= byteCount, "argument #1 to BitBuffer.setPointerByte should be within range of the buffer")
-- Sets the value of the pointer in bytes instead of bits
pointer = n * 8
pointerByte = n
end
local function setPointerByteFromEnd(n)
assert(type(n) == "number", "argument #1 to BitBuffer.setPointerByteFromEnd should be a number")
assert(n >= 0, "argument #1 to BitBuffer.setPointerByteFromEnd should be zero or higher")
assert(n % 1 == 0, "argument #1 to BitBuffer.setPointerByteFromEnd should be an integer")
assert(n <= byteCount, "argument #1 to BitBuffer.setPointerByteFromEnd should be within range of the buffer")
pointerByte = byteCount - n
pointer = pointerByte * 8
end
local function isFinished()
return pointer == bitCount
end
local function writeBits(...)
-- The first of two main functions for the actual 'writing' of the bitbuffer.
-- This function takes a vararg of 1s and 0s and writes them to the buffer.
local bitN = select("#", ...)
if bitN == 0 then
return
end -- Throwing here seems unnecessary
bitCount = bitCount + bitN
local packed = table.pack(...)
for _, v in ipairs(packed) do
assert(v == 1 or v == 0, "arguments to BitBuffer.writeBits should be either 1 or 0")
if bits == 0 then -- If the bit count is 0, increment the byteCount
-- This is the case at the beginning of the buffer as well as when the the buffer reaches 7 bits,
-- so it's done at the beginning of the loop.
byteCount = byteCount + 1
end
lastByte = lastByte + (v == 1 and powers_of_2[7 - bits] or 0) -- Add the current bit to lastByte, from right to left
bits = bits + 1
if bits == 8 then -- If the bit count is 8, set it to 0, write lastByte to the byte list, and set lastByte to 0
bits = 0
bytes[byteCount] = lastByte
lastByte = 0
end
end
if bits ~= 0 then -- If there are some bits in lastByte, it has to be put into lastByte
-- If this is done regardless of the bit count, there might be a trailing zero byte
bytes[byteCount] = lastByte
end
end
local function writeByte(n)
--assert(type(n) == "number", "argument #1 to BitBuffer.writeByte should be a number")
--assert(n >= 0 and n <= 255, "argument #1 to BitBuffer.writeByte should be in the range [0, 255]")
--assert(n % 1 == 0, "argument #1 to BitBuffer.writeByte should be an integer")
-- The second of two main functions for the actual 'writing' of the bitbuffer.
-- This function takes a byte (an 8-bit integer) and writes it to the buffer.
if bits == 0 then
-- If there aren't any free-floating bits, this is easy.
byteCount = byteCount + 1
bytes[byteCount] = n
else
local nibble = bit32.rshift(n, bits) -- Shift `bits` number of bits out of `n` (they go into the aether)
bytes[byteCount] = lastByte + nibble -- Manually set the most recent byte to the lastByte + the front part of `n`
byteCount = byteCount + 1
lastByte = bit32.band(bit32.lshift(n, 8 - bits), 255) -- Shift `n` forward `8-bits` and get what remains in the first 8 bits
bytes[byteCount] = lastByte
end
bitCount = bitCount + 8 -- Increment the bit counter
end
local function writeBytesFast(tab)
assert(bits == 0, "writeBytesFast can only work for whole byte streams")
local count = #tab
table.move(tab, 1 , count, byteCount + 1, bytes)
byteCount+= count
bitCount += count * 8
end
local function writeUnsigned(width, n)
assert(type(width) == "number", "argument #1 to BitBuffer.writeUnsigned should be a number")
assert(width >= 1 and width <= 64, "argument #1 to BitBuffer.writeUnsigned should be in the range [1, 64]")
assert(width % 1 == 0, "argument #1 to BitBuffer.writeUnsigned should be an integer")
assert(type(n) == "number", "argument #2 to BitBuffer.writeUnsigned should be a number")
assert(n >= 0 and n <= powers_of_2[width] - 1, "argument #2 to BitBuffer.writeUnsigned is out of range")
assert(n % 1 == 0, "argument #2 to BitBuffer.writeUnsigned should be an integer")
-- Writes unsigned integers of arbitrary length to the buffer.
-- This is the first function that uses other functions in the buffer to function.
-- This is done because the space taken up would be rather large for very little performance gain.
-- Get the number of bytes and number of floating bits in the specified width
local bytesInN, bitsInN = math.floor(width / 8), width % 8
local extractedBits = table.create(bitsInN) --!
-- If the width is less than or equal to 32-bits, bit32 can be used without any problem.
if width <= 32 then
-- Counting down from the left side, the bytes are written to the buffer
local c = width
for _ = 1, bytesInN do
c = c - 8
writeByte(bit32.extract(n, c, 8))
end
-- Any remaining bits are stored in an array
for i = bitsInN - 1, 0, -1 do
extractedBits[bitsInN - i] = BOOL_TO_BIT[bit32.btest(n, powers_of_2[i])]
end
-- Said array is then used to write them to the buffer
writeBits(table.unpack(extractedBits))
else
-- If the width is greater than 32, the number has to be divided up into a few 32-bit or less numbers
local leastSignificantChunk = n % 0x100000000 -- Get bits 0-31 (counting from the right side). 0x100000000 is 2^32.
local mostSignificantChunk = math.floor(n / 0x100000000) -- Get any remaining bits by manually right shifting by 32 bits
local c = width - 32 -- The number of bits in mostSignificantChunk is variable, but a counter is still needed
for _ = 1, bytesInN - 4 do -- 32 bits is 4 bytes
c = c - 8
writeByte(bit32.extract(mostSignificantChunk, c, 8))
end
-- `bitsInN` is always going to be the number of spare bits in `mostSignificantChunk`
-- which comes before `leastSignificantChunk`
for i = bitsInN - 1, 0, -1 do
extractedBits[bitsInN - i] = BOOL_TO_BIT[bit32.btest(mostSignificantChunk, powers_of_2[i])]
end
writeBits(table.unpack(extractedBits))
for i = 3, 0, -1 do -- Then of course, write all 4 bytes of leastSignificantChunk
writeByte(bit32.extract(leastSignificantChunk, i * 8, 8))
end
end
end
local function writeSigned(width, n)
assert(type(width) == "number", "argument #1 to BitBuffer.writeSigned should be a number")
assert(width >= 2 and width <= 64, "argument #1 to BitBuffer.writeSigned should be in the range [2, 64]")
assert(width % 1 == 0, "argument #1 to BitBuffer.writeSigned should be an integer")
assert(type(n) == "number", "argument #2 to BitBuffer.writeSigned should be a number")
assert(
n >= -powers_of_2[width - 1] and n <= powers_of_2[width - 1] - 1,
"argument #2 to BitBuffer.writeSigned is out of range"
)
assert(n % 1 == 0, "argument #2 to BitBuffer.writeSigned should be an integer")
-- Writes signed integers of arbitrary length to the buffer.
-- These integers are stored using two's complement.
-- Essentially, this means the first bit in the number is used to store whether it's positive or negative
-- If the number is positive, it's stored normally.
-- If it's negative, the number that's stored is equivalent to the max value of the width + the number
if n >= 0 then
writeBits(0)
writeUnsigned(width - 1, n) -- One bit is used for the sign, so the stored number's width is actually width-1
else
writeBits(1)
writeUnsigned(width - 1, powers_of_2[width - 1] + n)
end
end
local function writeFloat(exponentWidth, mantissaWidth, n)
assert(type(exponentWidth) == "number", "argument #1 to BitBuffer.writeFloat should be a number")
assert(
exponentWidth >= 1 and exponentWidth <= 64,
"argument #1 to BitBuffer.writeFloat should be in the range [1, 64]"
)
assert(exponentWidth % 1 == 0, "argument #1 to BitBuffer.writeFloat should be an integer")
assert(type(mantissaWidth) == "number", "argument #2 to BitBuffer.writeFloat should be a number")
assert(
mantissaWidth >= 1 and mantissaWidth <= 64,
"argument #2 to BitBuffer.writeFloat should be in the range [1, 64]"
)
assert(mantissaWidth % 1 == 0, "argument #2 to BitBuffer.writeFloat should be an integer")
assert(type(n) == "number", "argument #3 to BitBuffer.writeFloat should be a number")
-- Given that floating point numbers are particularly hard to grasp, this function is annotated heavily.
-- This stackoverflow answer is a great help if you just want an overview:
-- https://stackoverflow.com/a/7645264
-- Essentially, floating point numbers are scientific notation in binary.
-- Instead of expressing numbers like 10^e*m, floating points instead use 2^e*m.
-- For the sake of this function, `e` is referred to as `exponent` and `m` is referred to as `mantissa`.
-- Floating point numbers are stored in memory as a sequence of bitfields.
-- Every float has a set number of bits assigned for exponent values and mantissa values, along with one bit for the sign.
-- The order of the bits in the memory is: sign, exponent, mantissa.
-- Given that floating points have to represent numbers less than zero as well as those above them,
-- some parts of the exponent are set aside to be negative exponents. In the case of floats,
-- this is about half of the values. To calculate the 'real' value of an exponent a number that's half of the max exponent
-- is added to the exponent. More info can be found here: https://stackoverflow.com/q/2835278
-- This number is called the 'bias'.
local bias = powers_of_2[exponentWidth - 1] - 1
local sign = n < 0 -- The sign of a number is important.
-- In this case, since we're using a lookup table for the sign bit, we want `sign` to indicate if the number is negative or not.
n = math.abs(n) -- But it's annoying to work with negative numbers and the sign isn't important for decomposition.
-- Lua has a function specifically for decomposing (or taking apart) a floating point number into its pieces.
-- These pieces, as listed above, are the mantissa and exponent.
local mantissa, exponent = math.frexp(n)
-- Before we go further, there are some concepts that get special treatment in the floating point format.
-- These have to be accounted for before normal floats are written to the buffer.
if n == math.huge then
-- Positive and negative infinities are specifically indicated with an exponent that's all 1s
-- and a mantissa that's all 0s.
writeBits(BOOL_TO_BIT[sign]) -- As previously said, there's a bit for the sign
writeUnsigned(exponentWidth, powers_of_2[exponentWidth] - 1) -- Then comes the exponent
writeUnsigned(mantissaWidth, 0) -- And finally the mantissa
return
elseif n ~= n then
-- NaN is indicated with an exponent that's all 1s and a mantissa that isn't 0.
-- In theory, the individual bits of NaN should be maintained but Lua doesn't allow that,
-- so the mantissa is just being set to 10 for no particular reason.
writeBits(BOOL_TO_BIT[sign])
writeUnsigned(exponentWidth, powers_of_2[exponentWidth] - 1)
writeUnsigned(mantissaWidth, 10)
return
elseif n == 0 then
-- Zero is represented with an exponent that's zero and a mantissa that's also zero.
-- Lua doesn't have a signed zero, so that translates to the entire number being all 0s.
writeUnsigned(exponentWidth + mantissaWidth + 1, 0)
return
elseif exponent + bias <= 1 then
-- Subnormal numbers are a number that's exponent (when biased) is zero.
-- Because of a quirk with the way Lua and C decompose numbers, subnormal numbers actually have an exponent of one when biased.
-- The process behind this is explained below, so for the sake of brevity it isn't explained here.
-- The only difference between processing subnormal and normal numbers is with the mantissa.
-- As subnormal numbers always start with a 0 (in binary), it doesn't need to be removed or shifted out
-- so it's a simple shift and round.
mantissa = math.floor(mantissa * powers_of_2[mantissaWidth] + 0.5)
writeBits(BOOL_TO_BIT[sign])
writeUnsigned(exponentWidth, 0) -- Subnormal numbers always have zero for an exponent
writeUnsigned(mantissaWidth, mantissa)
return
end
-- In every normal case, the mantissa of a number will have a 1 directly after the decimal point (in binary).
-- As an example, 0.15625 has a mantissa of 0.625, which is 0.101 in binary. The 1 after the decimal point is always there.
-- That means that for the sake of space efficiency that can be left out.
-- The bit has to be removed. This uses subtraction and multiplication to do it since bit32 is for integers only.
-- The mantissa is then shifted up by the width of the mantissa field and rounded.
mantissa = math.floor((mantissa - 0.5) * 2 * powers_of_2[mantissaWidth] + 0.5)
-- (The first fraction bit is equivalent to 0.5 in decimal)
-- After that, it's just a matter of writing to the stream:
writeBits(BOOL_TO_BIT[sign])
writeUnsigned(exponentWidth, exponent + bias - 1) -- The bias is added to the exponent to properly offset it
-- The extra -1 is added because Lua, for whatever reason, doesn't normalize its results
-- This is the cause of the 'quirk' mentioned when handling subnormal number
-- As an example, math.frexp(0.15625) = 0.625, -2
-- This means that 0.15625 = 0.625*2^-2
-- Or, in binary: 0.00101 = 0.101 >> 2
-- This is a correct statement but the actual result is meant to be:
-- 0.00101 = 1.01 >> 3, or 0.15625 = 1.25*2^-3
-- A small but important distinction that has made writing this module frustrating because no documentation notates this.
writeUnsigned(mantissaWidth, mantissa)
end
local function writeBase64(input)
assert(type(input) == "string", "argument #1 to BitBuffer.writeBase64 should be a string")
assert(
not string.find(input, "[^%w%+/=]"),
"argument #1 to BitBuffer.writeBase64 should only contain valid base64 characters"
)
for i = 1, #input, 4 do
local b1, b2, b3, b4 = string.byte(input, i, i + 3)
b1 = decode_char_set[b1]
b2 = decode_char_set[b2]
b3 = decode_char_set[b3]
b4 = decode_char_set[b4]
local packed = bit32.bor(bit32.lshift(b1, 18), bit32.lshift(b2, 12), bit32.lshift(b3 or 0, 6), b4 or 0)
writeByte(bit32.rshift(packed, 16))
if not b3 then
break
end
writeByte(bit32.band(bit32.rshift(packed, 8), 0xff))
if not b4 then
break
end
writeByte(bit32.band(packed, 0xff))
end
end
local function writeString(str)
assert(type(str) == "string", "argument #1 to BitBuffer.writeString should be a string")
-- The default mode of writing strings is length-prefixed.
-- This means that the length of the string is written before the contents of the string.
-- For the sake of speed it has to be an even byte.
-- One and two bytes is too few characters (255 bytes and 65535 bytes respectively), so it has to be higher.
-- Three bytes is roughly 16.77mb, and four is roughly 4.295gb. Given this is Lua and is thus unlikely to be processing strings
-- that large, this function uses three bytes, or 24 bits for the length
writeUnsigned(24, #str)
for i = 1, #str do
writeByte(string.byte(str, i, i))
end
end
local function writeTerminatedString(str)
assert(type(str) == "string", "argument #1 to BitBuffer.writeTerminatedString should be a string")
-- This function writes strings that are null-terminated.
-- Null-terminated strings are strings of bytes that end in a 0 byte (\0)
-- This isn't the default because it doesn't allow for binary data to be written cleanly.
for i = 1, #str do
writeByte(string.byte(str, i, i))
end
writeByte(0)
end
local function writeSetLengthString(str)
assert(type(str) == "string", "argument #1 to BitBuffer.writeSetLengthString should be a string")
-- This function writes strings as a pure string of bytes
-- It doesn't store any data about the length of the string,
-- so reading it requires knowledge of how many characters were stored
for i = 1, #str do
writeByte(string.byte(str, i, i))
end
end
local function writeField(...)
-- This is equivalent to having a writeBitfield function.
-- It combines all of the passed 'bits' into an unsigned number, then writes it.
local field = 0
local bools = table.pack(...)
for i = 1, bools.n do
field = field * 2 -- Shift `field`. Equivalent to field<<1. At the beginning of the loop to avoid an extra shift.
local v = bools[i]
if v then
field = field + 1 -- If the bit is truthy, turn it on (it defaults to off so it's fine to not have a branch)
end
end
writeUnsigned(bools.n, field)
end
-- All write functions below here are shorthands. For the sake of performance, these functions are implemented manually.
-- As an example, while it would certainly be easier to make `writeInt16(n)` just call `writeUnsigned(16, n),
-- it's more performant to just manually call writeByte twice for it.
local function writeUInt8(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeUInt8 should be a number")
assert(n >= 0 and n <= 255, "argument #1 to BitBuffer.writeUInt8 should be in the range [0, 255]")
assert(n % 1 == 0, "argument #1 to BitBuffer.writeUInt8 should be an integer")
writeByte(n)
end
local function writeUInt16(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeUInt16 should be a number")
assert(n >= 0 and n <= 65535, "argument #1 to BitBuffer.writeInt16 should be in the range [0, 65535]")
assert(n % 1 == 0, "argument #1 to BitBuffer.writeUInt16 should be an integer")
writeByte(bit32.rshift(n, 8))
writeByte(bit32.band(n, 255))
end
local function writeUInt32(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeUInt32 should be a number")
assert(
n >= 0 and n <= 4294967295,
"argument #1 to BitBuffer.writeUInt32 should be in the range [0, 4294967295]"
)
assert(n % 1 == 0, "argument #1 to BitBuffer.writeUInt32 should be an integer")
writeByte(bit32.rshift(n, 24))
writeByte(bit32.band(bit32.rshift(n, 16), 255))
writeByte(bit32.band(bit32.rshift(n, 8), 255))
writeByte(bit32.band(n, 255))
end
local function writeInt8(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeInt8 should be a number")
assert(n >= -128 and n <= 127, "argument #1 to BitBuffer.writeInt8 should be in the range [-128, 127]")
assert(n % 1 == 0, "argument #1 to BitBuffer.writeInt8 should be an integer")
if n < 0 then
n = (128 + n) + 128
end
writeByte(n)
end
local function writeInt16(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeInt16 should be a number")
assert(n >= -32768 and n <= 32767, "argument #1 to BitBuffer.writeInt16 should be in the range [-32768, 32767]")
assert(n % 1 == 0, "argument #1 to BitBuffer.writeInt16 should be an integer")
if n < 0 then
n = (32768 + n) + 32768
end
writeByte(bit32.rshift(n, 8))
writeByte(bit32.band(n, 255))
end
local function writeInt32(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeInt32 should be a number")
assert(
n >= -2147483648 and n <= 2147483647,
"argument #1 to BitBuffer.writeInt32 should be in the range [-2147483648, 2147483647]"
)
assert(n % 1 == 0, "argument #1 to BitBuffer.writeInt32 should be an integer")
if n < 0 then
n = (2147483648 + n) + 2147483648
end
writeByte(bit32.rshift(n, 24))
writeByte(bit32.band(bit32.rshift(n, 16), 255))
writeByte(bit32.band(bit32.rshift(n, 8), 255))
writeByte(bit32.band(n, 255))
end
local function writeFloat16(n)
--assert(type(n) == "number", "argument #1 to BitBuffer.writeFloat16 should be a number")
local sign = n < 0
n = math.abs(n)
local mantissa, exponent = math.frexp(n)
if n == math.huge then
if sign then
writeByte(252) -- 11111100
else
writeByte(124) -- 01111100
end
writeByte(0) -- 00000000
return
elseif n ~= n then
-- 01111111 11111111
writeByte(127)
writeByte(255)
return
elseif n == 0 then
writeByte(0)
writeByte(0)
return
elseif exponent + 15 <= 1 then -- Bias for halfs is 15
mantissa = math.floor(mantissa * 1024 + 0.5)
if sign then
writeByte(128 + bit32.rshift(mantissa, 8)) -- Sign bit, 5 empty bits, 2 from mantissa
else
writeByte(bit32.rshift(mantissa, 8))
end
writeByte(bit32.band(mantissa, 255)) -- Get last 8 bits from mantissa
return
end
mantissa = math.floor((mantissa - 0.5) * 2048 + 0.5)
-- The bias for halfs is 15, 15-1 is 14
if sign then
writeByte(128 + bit32.lshift(exponent + 14, 2) + bit32.rshift(mantissa, 8))
else
writeByte(bit32.lshift(exponent + 14, 2) + bit32.rshift(mantissa, 8))
end
writeByte(bit32.band(mantissa, 255))
end
local function writeFloat32(n)
--assert(type(n) == "number", "argument #1 to BitBuffer.writeFloat32 should be a number")
local sign = n < 0
n = math.abs(n)
local mantissa, exponent = math.frexp(n)
if n == math.huge then
if sign then
writeByte(255) -- 11111111
else
writeByte(127) -- 01111111
end
writeByte(128) -- 10000000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
return
elseif n ~= n then
-- 01111111 11111111 11111111 11111111
writeByte(127)
writeByte(255)
writeByte(255)
writeByte(255)
return
elseif n == 0 then
writeByte(0)
writeByte(0)
writeByte(0)
writeByte(0)
return
elseif exponent + 127 <= 1 then -- bias for singles is 127
mantissa = math.floor(mantissa * 8388608 + 0.5)
if sign then
writeByte(128) -- Sign bit, 7 empty bits for exponent
else
writeByte(0)
end
writeByte(bit32.rshift(mantissa, 16))
writeByte(bit32.band(bit32.rshift(mantissa, 8), 255))
writeByte(bit32.band(mantissa, 255))
return
end
mantissa = math.floor((mantissa - 0.5) * 16777216 + 0.5)
-- 127-1 = 126
if sign then -- sign + 7 exponent
writeByte(128 + bit32.rshift(exponent + 126, 1))
else
writeByte(bit32.rshift(exponent + 126, 1))
end
writeByte(bit32.band(bit32.lshift(exponent + 126, 7), 255) + bit32.rshift(mantissa, 16)) -- 1 exponent + 7 mantissa
writeByte(bit32.band(bit32.rshift(mantissa, 8), 255)) -- 8 mantissa
writeByte(bit32.band(mantissa, 255)) -- 8 mantissa
end
local function writeFloat64(n)
assert(type(n) == "number", "argument #1 to BitBuffer.writeFloat64 should be a number")
local sign = n < 0
n = math.abs(n)
local mantissa, exponent = math.frexp(n)
if n == math.huge then
if sign then
writeByte(255) -- 11111111
else
writeByte(127) -- 01111111
end
writeByte(240) -- 11110000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
writeByte(0) -- 00000000
return
elseif n ~= n then
-- 01111111 11111111 11111111 11111111 11111111 11111111 11111111 11111111
writeByte(127)
writeByte(255)
writeByte(255)
writeByte(255)
writeByte(255)
writeByte(255)
writeByte(255)
writeByte(255)
return
elseif n == 0 then
writeByte(0)
return
elseif exponent + 1023 <= 1 then -- bias for doubles is 1023
mantissa = math.floor(mantissa * 4503599627370496 + 0.5)
if sign then
writeByte(128) -- Sign bit, 7 empty bits for exponent
else
writeByte(0)
end
-- This is labeled better below
local leastSignificantChunk = mantissa % 0x100000000 -- 32 bits
local mostSignificantChunk = math.floor(mantissa / 0x100000000) -- 20 bits
writeByte(bit32.rshift(mostSignificantChunk, 16))
writeByte(bit32.band(bit32.rshift(mostSignificantChunk, 8), 255))
writeByte(bit32.band(mostSignificantChunk, 255))
writeByte(bit32.rshift(leastSignificantChunk, 24))
writeByte(bit32.band(bit32.rshift(leastSignificantChunk, 16), 255))
writeByte(bit32.band(bit32.rshift(leastSignificantChunk, 8), 255))
writeByte(bit32.band(leastSignificantChunk, 255))
return
end
mantissa = math.floor((mantissa - 0.5) * 9007199254740992 + 0.5)
--1023-1 = 1022
if sign then
writeByte(128 + bit32.rshift(exponent + 1022, 4)) -- shift out 4 of the bits in exponent
else
writeByte(bit32.rshift(exponent + 1022, 4)) -- 01000001 0110
end
-- Things start to get a bit wack here because the mantissa is 52 bits, so bit32 *can't* be used.
-- As the Offspring once said... You gotta keep 'em seperated.
local leastSignificantChunk = mantissa % 0x100000000 -- 32 bits
local mostSignificantChunk = math.floor(mantissa / 0x100000000) -- 20 bits
-- First, the last 4 bits of the exponent and the first 4 bits of the mostSignificantChunk:
writeByte(bit32.band(bit32.lshift(exponent + 1022, 4), 255) + bit32.rshift(mostSignificantChunk, 16))
-- Then, the next 16 bits:
writeByte(bit32.band(bit32.rshift(mostSignificantChunk, 8), 255))
writeByte(bit32.band(mostSignificantChunk, 255))
-- Then... 4 bytes of the leastSignificantChunk
writeByte(bit32.rshift(leastSignificantChunk, 24))
writeByte(bit32.band(bit32.rshift(leastSignificantChunk, 16), 255))
writeByte(bit32.band(bit32.rshift(leastSignificantChunk, 8), 255))
writeByte(bit32.band(leastSignificantChunk, 255))
end
-- All write functions below here are Roblox specific datatypes.
local function writeBrickColor(n)
assert(typeof(n) == "BrickColor", "argument #1 to BitBuffer.writeBrickColor should be a BrickColor")
writeUInt16(n.Number)
end
local function writeColor3(c3)
assert(typeof(c3) == "Color3", "argument #1 to BitBuffer.writeColor3 should be a Color3")
writeByte(math.floor(c3.R * 0xff + 0.5))
writeByte(math.floor(c3.G * 0xff + 0.5))
writeByte(math.floor(c3.B * 0xff + 0.5))
end
local function writeCFrame(cf)
assert(typeof(cf) == "CFrame", "argument #1 to BitBuffer.writeCFrame should be a CFrame")
-- CFrames can be rather lengthy (if stored naively, they would each be 48 bytes long) so some optimization is done here.
-- Specifically, if a CFrame is axis-aligned (it's only rotated in 90 degree increments), the rotation matrix isn't stored.
-- Instead, an 'id' for its orientation is generated and that's stored instead of the rotation.
-- This means that for the most common rotations, only 13 bytes are used.
-- The downside is that non-axis-aligned CFrames use 49 bytes instead of 48, but that's a small price to pay.
local upVector = cf.UpVector
local rightVector = cf.RightVector
-- This is an easy trick to check if a CFrame is axis-aligned:
-- Essentially, in order for a vector to be axis-aligned, two of the components have to be 0
-- This means that the dot product between the vector and a vector of all 1s will be 1 (0*x = 0)
-- Since these are all unit vectors, there is no other combination that results in 1.
local rightAligned = math.abs(rightVector:Dot(ONES_VECTOR))
local upAligned = math.abs(upVector:Dot(ONES_VECTOR))
-- At least one of these two vectors is guaranteed to not result in 0.
local axisAligned = (math.abs(1 - rightAligned) < 0.00001 or rightAligned == 0)
and (math.abs(1 - upAligned) < 0.00001 or upAligned == 0)
-- There are limitations to `math.abs(a-b) < epsilon` but they're not relevant:
-- The range of numbers is [0, 1] and this just needs to know if the number is approximately 1
--todo special code for quaternions (0x01 in Roblox's format, would clash with 0x00 here)
if axisAligned then
local position = cf.Position
-- The ID of an orientation is generated through what can best be described as 'hand waving';
-- This is how Roblox does it and it works, so it was chosen to do it this way too.
local rightNormal, upNormal
for i = 0, 5 do
local v = NORMAL_ID_VECTORS[i]
if 1 - v:Dot(rightVector) < 0.00001 then
rightNormal = i
end
if 1 - v:Dot(upVector) < 0.00001 then
upNormal = i
end
end
-- The ID generated here is technically off by 1 from what Roblox would store, but that's not important
-- It just means that 0x02 is actually 0x01 for the purposes of this module's implementation.
writeByte(rightNormal * 6 + upNormal)
writeFloat32(position.X)
writeFloat32(position.Y)
writeFloat32(position.Z)
else
-- If the CFrame isn't axis-aligned, the entire rotation matrix has to be written...
writeByte(0) -- Along with a byte to indicate the matrix was written.
local x, y, z, r00, r01, r02, r10, r11, r12, r20, r21, r22 = cf:GetComponents()
writeFloat32(x)
writeFloat32(y)
writeFloat32(z)
writeFloat32(r00)
writeFloat32(r01)
writeFloat32(r02)
writeFloat32(r10)
writeFloat32(r11)
writeFloat32(r12)
writeFloat32(r20)
writeFloat32(r21)
writeFloat32(r22)
end
end
local function writeVector3(v3)
--assert(typeof(v3) == "Vector3", "argument #1 to BitBuffer.writeVector3 should be a Vector3")
writeFloat32(v3.X)
writeFloat32(v3.Y)
writeFloat32(v3.Z)
end
local function writeVector2(v2)
assert(typeof(v2) == "Vector2", "argument #1 to BitBuffer.writeVector2 should be a Vector2")
writeFloat32(v2.X)
writeFloat32(v2.Y)
end
local function writeUDim2(u2)
assert(typeof(u2) == "UDim2", "argument #1 to BitBuffer.writeUDim2 should be a UDim2")
writeFloat32(u2.X.Scale)
writeInt32(u2.X.Offset)
writeFloat32(u2.Y.Scale)
writeInt32(u2.Y.Offset)
end
local function writeUDim(u)
assert(typeof(u) == "UDim", "argument #1 to BitBuffer.writeUDim should be a UDim")
writeFloat32(u.Scale)
writeInt32(u.Offset)
end
local function writeRay(ray)
assert(typeof(ray) == "Ray", "argument #1 to BitBuffer.writeRay should be a Ray")
writeFloat32(ray.Origin.X)
writeFloat32(ray.Origin.Y)
writeFloat32(ray.Origin.Z)
writeFloat32(ray.Direction.X)
writeFloat32(ray.Direction.Y)
writeFloat32(ray.Direction.Z)
end
local function writeRect(rect)
assert(typeof(rect) == "Rect", "argument #1 to BitBuffer.writeRect should be a Rect")
writeFloat32(rect.Min.X)
writeFloat32(rect.Min.Y)
writeFloat32(rect.Max.X)
writeFloat32(rect.Max.Y)
end
local function writeRegion3(region)
assert(typeof(region) == "Region3", "argument #1 to BitBuffer.writeRegion3 should be a Region3")
local min = region.CFrame.Position - (region.Size / 2)
local max = region.CFrame.Position + (region.Size / 2)
writeFloat32(min.X)
writeFloat32(min.Y)
writeFloat32(min.Z)
writeFloat32(max.X)
writeFloat32(max.Y)
writeFloat32(max.Z)
end
local function writeEnum(enum)
assert(typeof(enum) == "EnumItem", "argument #1 to BitBuffer.writeEnum should be an EnumItem")
-- Relying upon tostring is generally not good, but there's not any other options for this.
writeTerminatedString(tostring(enum.EnumType))
writeUInt16(enum.Value) -- Optimistically assuming no Roblox Enum value will ever pass 65,535
end
local function writeNumberRange(range)
assert(typeof(range) == "NumberRange", "argument #1 to BitBuffer.writeNumberRange should be a NumberRange")
writeFloat32(range.Min)
writeFloat32(range.Max)
end
local function writeNumberSequence(sequence)
assert(
typeof(sequence) == "NumberSequence",
"argument #1 to BitBuffer.writeNumberSequence should be a NumberSequence"
)
writeUInt32(#sequence.Keypoints)
for _, keypoint in ipairs(sequence.Keypoints) do
writeFloat32(keypoint.Time)
writeFloat32(keypoint.Value)
writeFloat32(keypoint.Envelope)
end
end
local function writeColorSequence(sequence)
assert(
typeof(sequence) == "ColorSequence",
"argument #1 to BitBuffer.writeColorSequence should be a ColorSequence"
)
writeUInt32(#sequence.Keypoints)
for _, keypoint in ipairs(sequence.Keypoints) do
local c3 = keypoint.Value
writeFloat32(keypoint.Time)
writeByte(math.floor(c3.R * 0xff + 0.5))
writeByte(math.floor(c3.G * 0xff + 0.5))
writeByte(math.floor(c3.B * 0xff + 0.5))
end
end
-- These are the read functions for the 'abstract' data types. At the bottom, there are shorthand read functions.
local function readBits(n)
assert(type(n) == "number", "argument #1 to BitBuffer.readBits should be a number")
assert(n > 0, "argument #1 to BitBuffer.readBits should be greater than zero")
assert(n % 1 == 0, "argument #1 to BitBuffer.readBits should be an integer")
assert(pointer + n <= bitCount, "BitBuffer.readBits cannot read past the end of the stream")
-- The first of two main functions for the actual 'reading' of the bitbuffer.
-- Reads `n` bits and returns an array of their values.
local output = table.create(n) --!
local byte = bytes[pointerByte] -- For the sake of efficiency, the current byte that the bits are coming from is stored
local c = pointer % 8 -- A counter is set with the current position of the pointer in the byte
for i = 1, n do
-- Then, it's as easy as moving through the bits of the byte
-- And getting the individiual bit values
local pow = powers_of_2[7 - c]
output[i] = BOOL_TO_BIT[bit32.btest(byte, pow)] -- Test if a bit is on by &ing it by 2^[bit position]
c = c + 1
if c == 8 then -- If the byte boundary is reached, increment pointerByte and store the new byte in `byte`
pointerByte = pointerByte + 1
byte = bytes[pointerByte]
c = 0
end
end
pointer = pointer + n -- Move the pointer forward
return output
end
--Skip to the end of the current byte
local function skipStrayBits()
local c = pointer % 8
if (c > 0) then
pointer += 8 - c
pointerByte += 1
end
end
local function readBytesFast()
return bytes
end
local function readByte()
assert(pointer + 8 <= bitCount, "BitBuffer.readByte cannot read past the end of the stream")
-- The second of two main functions for the actual 'reading' of the bitbuffer.
-- Reads a byte and returns it
local c = pointer % 8 -- How far into the pointerByte the pointer is
local byte1 = bytes[pointerByte] -- The pointerByte
pointer = pointer + 8
if c == 0 then -- Trivial if the pointer is at the beginning of the pointerByte
pointerByte = pointerByte + 1
return byte1
else
pointerByte = pointerByte + 1
-- Get the remainder of the first pointerByte and add it to the part of the new pointerByte that's required
-- Both these methods are explained in writeByte
return bit32.band(bit32.lshift(byte1, c), 255) + bit32.rshift(bytes[pointerByte], 8 - c)
end
end
local function readUnsigned(width)
assert(type(width) == "number", "argument #1 to BitBuffer.readUnsigned should be a number")
assert(width >= 1 and width <= 64, "argument #1 to BitBuffer.readUnsigned should be in the range [1, 64]")
assert(width % 1 == 0, "argument #1 to BitBuffer.readUnsigned should be an integer")
assert(pointer + width <= bitCount, "BitBuffer.readUnsigned cannot read past the end of the stream")
-- Implementing this on its own was considered because of a worry that it would be inefficient to call
-- readByte and readBit several times, but it was decided the simplicity is worth a minor performance hit.
local bytesInN, bitsInN = math.floor(width / 8), width % 8
-- No check is required for if the width is greater than 32 because bit32 isn't used.
local n = 0
-- Shift and add a read byte however many times is necessary
-- Adding after shifting is importnat - it prevents there from being 8 empty bits of space
for _ = 1, bytesInN do
n = n * 0x100 -- 2^8; equivalent to n << 8
n = n + readByte()
end
-- The bits are then read and added to the number
if bitsInN ~= 0 then
for _, v in ipairs(readBits(width % 8)) do --todo benchmark against concat+tonumber; might be worth the code smell
n = n * 2
n = n + v
end
end
return n
end
local function readSigned(width)
assert(type(width) == "number", "argument #1 to BitBuffer.readSigned should be a number")
assert(width >= 2 and width <= 64, "argument #1 to BitBuffer.readSigned should be in the range [2, 64]")
assert(width % 1 == 0, "argument #1 to BitBuffer.readSigned should be an integer")
assert(pointer + 8 <= bitCount, "BitBuffer.readSigned cannot read past the end of the stream")
local sign = readBits(1)[1]
local n = readUnsigned(width - 1) -- Again, width-1 is because one bit is used for the sign
-- As said in writeSigned, the written number is unmodified if the number is positive (the sign bit is 0)
if sign == 0 then
return n
else
-- And the number is equal to max value of the width + the number if the number is negative (the sign bit is 1)
-- To reverse that, the max value is subtracted from the stored number.
return n - powers_of_2[width - 1]
end
end
local function readFloat(exponentWidth, mantissaWidth)
assert(type(exponentWidth) == "number", "argument #1 to BitBuffer.readFloat should be a number")
assert(
exponentWidth >= 1 and exponentWidth <= 64,
"argument #1 to BitBuffer.readFloat should be in the range [1, 64]"
)
assert(exponentWidth % 1 == 0, "argument #1 to BitBuffer.readFloat should be an integer")
assert(type(mantissaWidth) == "number", "argument #2 to BitBuffer.readFloat should be a number")
assert(
mantissaWidth >= 1 and mantissaWidth <= 64,
"argument #2 to BitBuffer.readFloat should be in the range [1, 64]"
)
assert(mantissaWidth % 1 == 0, "argument #2 to BitBuffer.readFloat should be an integer")
assert(
pointer + exponentWidth + mantissaWidth + 1 <= bitCount,
"BitBuffer.readFloat cannot read past the end of the stream"
)
-- Recomposing floats is rather straightfoward.
-- The bias is subtracted from the exponent, the mantissa is shifted back by mantissaWidth, one is added to the mantissa
-- and the whole thing is recomposed with math.ldexp (this is identical to mantissa*(2^exponent)).
local bias = powers_of_2[exponentWidth - 1] - 1
local sign = readBits(1)[1]
local exponent = readUnsigned(exponentWidth)
local mantissa = readUnsigned(mantissaWidth)
-- Before normal numbers are handled though, special cases and subnormal numbers are once again handled seperately
if exponent == powers_of_2[exponentWidth] - 1 then
if mantissa ~= 0 then -- If the exponent is all 1s and the mantissa isn't zero, the number is NaN
return 0 / 0
else -- Otherwise, it's positive or negative infinity
return sign == 0 and math.huge or -math.huge
end
elseif exponent == 0 then
if mantissa == 0 then -- If the exponent and mantissa are both zero, the number is zero.
return 0
else -- If the exponent is zero and the mantissa is not zero, the number is subnormal
-- Subnormal numbers are straightforward: shifting the mantissa so that it's a fraction is all that's required
mantissa = mantissa / powers_of_2[mantissaWidth]
-- Since the exponent is 0, it's actual value is just -bias (it would be exponent-bias)
-- As previously touched on in writeFloat, the exponent value is off by 1 in Lua though.
return sign == 1 and -math.ldexp(mantissa, -bias + 1) or math.ldexp(mantissa, -bias + 1)
end
end
-- First, the mantissa is shifted back by the mantissaWidth
-- Then, 1 is added to it to 'normalize' it.
mantissa = (mantissa / powers_of_2[mantissaWidth]) + 1
-- Because the mantissa is normalized above (the leading 1 is in the ones place), it's accurate to say exponent-bias
return sign == 1 and -math.ldexp(mantissa, exponent - bias) or math.ldexp(mantissa, exponent - bias)
end
local function readString()
assert(pointer + 24 <= bitCount, "BitBuffer.readString cannot read past the end of the stream")
-- Reading a length-prefixed string is rather straight forward.
-- The length is read, then that many bytes are read and put in a string.
local stringLength = readUnsigned(24)
assert(pointer + (stringLength * 8) <= bitCount, "BitBuffer.readString cannot read past the end of the stream")
local outputCharacters = table.create(stringLength) --!
for i = 1, stringLength do
outputCharacters[i] = readByte()
end
local output = table.create(math.ceil(stringLength / 4096))
local k = 1
for i = 1, stringLength, 4096 do
output[k] = string.char(table.unpack(outputCharacters, i, math.min(stringLength, i + 4095)))
k = k + 1
end
return table.concat(output)
end
local function readTerminatedString()
local outputCharacters = {}
-- Bytes are read continuously until either a nul-character is reached or until the stream runs out.
local length = 0
while true do
local byte = readByte()
if not byte then -- Stream has ended
error("BitBuffer.readTerminatedString cannot read past the end of the stream", 2)
elseif byte == 0 then -- String has ended
break
else -- Add byte to string
length = length + 1
outputCharacters[length] = byte
end
end
local output = table.create(math.ceil(length / 4096))
local k = 1
for l = 1, length, 4096 do
output[k] = string.char(table.unpack(outputCharacters, l, math.min(length, l + 4095)))
k = k + 1
end
return table.concat(output)
end
local function readSetLengthString(length)
assert(type(length) == "number", "argument #1 to BitBuffer.readSetLengthString should be a number")
assert(length >= 0, "argument #1 to BitBuffer.readSetLengthString should be zero or higher.")
assert(length % 1 == 0, "argument #1 to BitBuffer.readSetLengthString should be an integer")
assert(
pointer + (length * 8) <= bitCount,
"BitBuffer.readSetLengthString cannot read past the end of the stream"
)
-- `length` number of bytes are read and put into a string
local outputCharacters = table.create(length) --!
for i = 1, length do
outputCharacters[i] = readByte()
end
local output = table.create(math.ceil(length / 4096))
local k = 1
for i = 1, length, 4096 do
output[k] = string.char(table.unpack(outputCharacters, i, math.min(length, i + 4095)))
k = k + 1
end
return table.concat(output)
end
local function readField(n)
assert(type(n) == "number", "argument #1 to BitBuffer.readField should be a number")
assert(n > 0, "argument #1 to BitBuffer.readField should be above 0")
assert(n % 1 == 0, "argument #1 to BitBuffer.readField should be an integer")
assert(pointer + n <= bitCount, "BitBuffer.readField cannot read past the end of the stream")
-- Reading a bit field is again rather simple. You read the actual field, then take the bits out.
local readInt = readUnsigned(n)
local output = table.create(n) --!
for i = n, 1, -1 do -- In reverse order since we're pulling bits out from lsb to msb
output[i] = readInt % 2 == 1 -- Equivalent to an extraction of the lsb
readInt = math.floor(readInt / 2) -- Equivalent to readInt>>1
end
return output
end
-- All read functions below here are shorthands.
-- As with their write variants, these functions are implemented manually using readByte for performance reasons.
local function readUInt8()
assert(pointer + 8 <= bitCount, "BitBuffer.readUInt8 cannot read past the end of the stream")
return readByte()
end
local function readUInt16()
assert(pointer + 16 <= bitCount, "BitBuffer.readUInt16 cannot read past the end of the stream")
return bit32.lshift(readByte(), 8) + readByte()
end
local function readUInt32()
assert(pointer + 32 <= bitCount, "BitBuffer.readUInt32 cannot read past the end of the stream")
return bit32.lshift(readByte(), 24) + bit32.lshift(readByte(), 16) + bit32.lshift(readByte(), 8) + readByte()
end
local function readInt8()
assert(pointer + 8 <= bitCount, "BitBuffer.readInt8 cannot read past the end of the stream")
local n = readByte()
local sign = bit32.btest(n, 128)
n = bit32.band(n, 127)
if sign then
return n - 128
else
return n
end
end
local function readInt16()
assert(pointer + 16 <= bitCount, "BitBuffer.readInt16 cannot read past the end of the stream")
local n = bit32.lshift(readByte(), 8) + readByte()
local sign = bit32.btest(n, 32768)
n = bit32.band(n, 32767)
if sign then
return n - 32768
else
return n
end
end
local function readInt32()
assert(pointer + 32 <= bitCount, "BitBuffer.readInt32 cannot read past the end of the stream")
local n = bit32.lshift(readByte(), 24) + bit32.lshift(readByte(), 16) + bit32.lshift(readByte(), 8) + readByte()
local sign = bit32.btest(n, 2147483648)
n = bit32.band(n, 2147483647)
if sign then
return n - 2147483648
else
return n
end
end
local function readFloat16()
assert(pointer + 16 <= bitCount, "BitBuffer.readFloat16 cannot read past the end of the stream")
local b0 = readByte()
local sign = bit32.btest(b0, 128)
local exponent = bit32.rshift(bit32.band(b0, 127), 2)
local mantissa = bit32.lshift(bit32.band(b0, 3), 8) + readByte()
if exponent == 31 then --2^5-1
if mantissa ~= 0 then
return 0 / 0
else
return sign and -math.huge or math.huge
end
elseif exponent == 0 then
if mantissa == 0 then
return 0
else
return sign and -math.ldexp(mantissa / 1024, -14) or math.ldexp(mantissa / 1024, -14)
end
end
mantissa = (mantissa / 1024) + 1
return sign and -math.ldexp(mantissa, exponent - 15) or math.ldexp(mantissa, exponent - 15)
end
local function readFloat32()
assert(pointer + 32 <= bitCount, "BitBuffer.readFloat32 cannot read past the end of the stream")
local b0 = readByte()
local b1 = readByte()
local sign = bit32.btest(b0, 128)
local exponent = bit32.band(bit32.lshift(b0, 1), 255) + bit32.rshift(b1, 7)
local mantissa = bit32.lshift(bit32.band(b1, 127), 23 - 7)
+ bit32.lshift(readByte(), 23 - 7 - 8)
+ bit32.lshift(readByte(), 23 - 7 - 8 - 8)
if exponent == 255 then -- 2^8-1
if mantissa ~= 0 then
return 0 / 0
else
return sign and -math.huge or math.huge
end
elseif exponent == 0 then
if mantissa == 0 then
return 0
else
-- -126 is the 0-bias+1
return sign and -math.ldexp(mantissa / 8388608, -126) or math.ldexp(mantissa / 8388608, -126)
end
end
mantissa = (mantissa / 8388608) + 1
return sign and -math.ldexp(mantissa, exponent - 127) or math.ldexp(mantissa, exponent - 127)
end
local function readFloat64()
assert(pointer + 64 <= bitCount, "BitBuffer.readFloat64 cannot read past the end of the stream")
local b0 = readByte()
local b1 = readByte()
local sign = bit32.btest(b0, 128)
local exponent = bit32.lshift(bit32.band(b0, 127), 4) + bit32.rshift(b1, 4)
local mostSignificantChunk = bit32.lshift(bit32.band(b1, 15), 16) + bit32.lshift(readByte(), 8) + readByte()
local leastSignificantChunk = bit32.lshift(readByte(), 24)
+ bit32.lshift(readByte(), 16)
+ bit32.lshift(readByte(), 8)
+ readByte()
-- local mantissa = (bit32.lshift(bit32.band(b1, 15), 16)+bit32.lshift(readByte(), 8)+readByte())*0x100000000+
-- bit32.lshift(readByte(), 24)+bit32.lshift(readByte(), 16)+bit32.lshift(readByte(), 8)+readByte()
local mantissa = mostSignificantChunk * 0x100000000 + leastSignificantChunk
if exponent == 2047 then -- 2^11-1
if mantissa ~= 0 then
return 0 / 0
else
return sign and -math.huge or math.huge
end
elseif exponent == 0 then
if mantissa == 0 then
return 0
else
return sign and -math.ldexp(mantissa / 4503599627370496, -1022)
or math.ldexp(mantissa / 4503599627370496, -1022)
end
end
mantissa = (mantissa / 4503599627370496) + 1
return sign and -math.ldexp(mantissa, exponent - 1023) or math.ldexp(mantissa, exponent - 1023)
end
-- All read functions below here are Roblox specific datatypes.
local function readBrickColor()
assert(pointer + 16 <= bitCount, "BitBuffer.readBrickColor cannot read past the end of the stream")
return BrickColor.new(readUInt16())
end
local function readColor3()
assert(pointer + 24 <= bitCount, "BitBuffer.readColor3 cannot read past the end of the stream")
return Color3.fromRGB(readByte(), readByte(), readByte())
end
local function readCFrame()
assert(pointer + 8 <= bitCount, "BitBuffer.readCFrame cannot read past the end of the stream")
local id = readByte()
if id == 0 then
assert(pointer + 384 <= bitCount, "BitBuffer.readCFrame cannot read past the end of the stream") -- 4*12 bytes = 383 bits
-- stylua: ignore
return CFrame.new(
readFloat32(), readFloat32(), readFloat32(),
readFloat32(), readFloat32(), readFloat32(),
readFloat32(), readFloat32(), readFloat32(),
readFloat32(), readFloat32(), readFloat32()
)
else
assert(pointer + 96 <= bitCount, "BitBuffer.readCFrame cannot read past the end of the stream") -- 4*3 bytes = 96 bits
local rightVector = NORMAL_ID_VECTORS[math.floor(id / 6)]
local upVector = NORMAL_ID_VECTORS[id % 6]
local lookVector = rightVector:Cross(upVector)
-- CFrame's full-matrix constructor takes right/up/look vectors as columns...
-- stylua: ignore
return CFrame.new(
readFloat32(), readFloat32(), readFloat32(),
rightVector.X, upVector.X, lookVector.X,
rightVector.Y, upVector.Y, lookVector.Y,
rightVector.Z, upVector.Z, lookVector.Z
)
end
end
local function readVector3()
assert(pointer + 96 <= bitCount, "BitBuffer.readVector3 cannot read past the end of the stream")
return Vector3.new(readFloat32(), readFloat32(), readFloat32())
end
local function readVector2()
assert(pointer + 64 <= bitCount, "BitBuffer.readVector2 cannot read past the end of the stream")
return Vector2.new(readFloat32(), readFloat32())
end
local function readUDim2()
assert(pointer + 128 <= bitCount, "BitBuffer.readUDim2 cannot read past the end of the stream")
return UDim2.new(readFloat32(), readInt32(), readFloat32(), readInt32())
end
local function readUDim()
assert(pointer + 64 <= bitCount, "BitBuffer.readUDim cannot read past the end of the stream")
return UDim.new(readFloat32(), readInt32())
end
local function readRay()
assert(pointer + 192 <= bitCount, "BitBuffer.readRay cannot read past the end of the stream")
return Ray.new(
Vector3.new(readFloat32(), readFloat32(), readFloat32()),
Vector3.new(readFloat32(), readFloat32(), readFloat32())
)
end
local function readRect()
assert(pointer + 128 <= bitCount, "BitBuffer.readRect cannot read past the end of the stream")
return Rect.new(readFloat32(), readFloat32(), readFloat32(), readFloat32())
end
local function readRegion3()
assert(pointer + 192 <= bitCount, "BitBuffer.readRegion3 cannot read past the end of the stream")
return Region3.new(
Vector3.new(readFloat32(), readFloat32(), readFloat32()),
Vector3.new(readFloat32(), readFloat32(), readFloat32())
)
end
local function readEnum()
assert(pointer + 8 <= bitCount, "BitBuffer.readEnum cannot read past the end of the stream")
local name = readTerminatedString() -- This might expose an error from readString to the end-user but it's not worth the hassle to fix.
assert(pointer + 16 <= bitCount, "BitBuffer.readEnum cannot read past the end of the stream")
local value = readUInt16() -- Again, optimistically assuming no Roblox Enum value will ever pass 65,535
-- Catching a potential error only to throw it with different formatting seems... Superfluous.
-- Open an issue on github if you feel otherwise.
for _, v in ipairs(Enum[name]:GetEnumItems()) do
if v.Value == value then
return v
end
end
error(
"BitBuffer.readEnum could not get value: `"
.. tostring(value)
.. "` is not a valid member of `"
.. name
.. "`",
2
)
end
local function readNumberRange()
assert(pointer + 64 <= bitCount, "BitBuffer.readNumberRange cannot read past the end of the stream")
return NumberRange.new(readFloat32(), readFloat32())
end
local function readNumberSequence()
assert(pointer + 32 <= bitCount, "BitBuffer.readNumberSequence cannot read past the end of the stream")
local keypointCount = readUInt32()
assert(pointer + keypointCount * 96, "BitBuffer.readColorSequence cannot read past the end of the stream")
local keypoints = table.create(keypointCount)
-- As it turns out, creating a NumberSequence with a negative value as its first argument (in the first and second constructor)
-- creates NumberSequenceKeypoints with negative envelopes. The envelope is read and saved properly, as you would expect,
-- but you can't create a NumberSequence with a negative envelope if you're using a table of keypoints (which is happening here).
-- If you're confused, run this snippet: NumberSequence.new(NumberSequence.new(-1).Keypoints)
-- As a result, there has to be some branching logic in this function.
-- ColorSequences don't have envelopes so it's not necessary for them.
for i = 1, keypointCount do
local time, value, envelope = readFloat32(), readFloat32(), readFloat32()
if value < 0 then
envelope = nil
end
keypoints[i] = NumberSequenceKeypoint.new(time, value, envelope)
end
return NumberSequence.new(keypoints)
end
local function readColorSequence()
assert(pointer + 32 <= bitCount, "BitBuffer.readColorSequence cannot read past the end of the stream")
local keypointCount = readUInt32()
assert(pointer + keypointCount * 56, "BitBuffer.readColorSequence cannot read past the end of the stream")
local keypoints = table.create(keypointCount)
for i = 1, keypointCount do
keypoints[i] = ColorSequenceKeypoint.new(readFloat32(), Color3.fromRGB(readByte(), readByte(), readByte()))
end
return ColorSequence.new(keypoints)
end
return {
dumpBinary = dumpBinary,
dumpString = dumpString,
dumpHex = dumpHex,
dumpBase64 = dumpBase64,
exportChunk = exportChunk,
exportBase64Chunk = exportBase64Chunk,
exportHexChunk = exportHexChunk,
crc32 = crc32,
getLength = getLength,
getByteLength = getByteLength,
getPointer = getPointer,
setPointer = setPointer,
setPointerFromEnd = setPointerFromEnd,
getPointerByte = getPointerByte,
setPointerByte = setPointerByte,
setPointerByteFromEnd = setPointerByteFromEnd,
isFinished = isFinished,
writeBits = writeBits,
writeByte = writeByte,
writeBytesFast = writeBytesFast,
writeUnsigned = writeUnsigned,
writeSigned = writeSigned,
writeFloat = writeFloat,
writeBase64 = writeBase64,
writeString = writeString,
writeTerminatedString = writeTerminatedString,
writeSetLengthString = writeSetLengthString,
writeField = writeField,
writeUInt8 = writeUInt8,
writeUInt16 = writeUInt16,
writeUInt32 = writeUInt32,
writeInt8 = writeInt8,
writeInt16 = writeInt16,
writeInt32 = writeInt32,
writeFloat16 = writeFloat16,
writeFloat32 = writeFloat32,
writeFloat64 = writeFloat64,
writeBrickColor = writeBrickColor,
writeColor3 = writeColor3,
writeCFrame = writeCFrame,
writeVector3 = writeVector3,
writeVector2 = writeVector2,
writeUDim2 = writeUDim2,
writeUDim = writeUDim,
writeRay = writeRay,
writeRect = writeRect,
writeRegion3 = writeRegion3,
writeEnum = writeEnum,
writeNumberRange = writeNumberRange,
writeNumberSequence = writeNumberSequence,
writeColorSequence = writeColorSequence,
readBits = readBits,
readByte = readByte,
readUnsigned = readUnsigned,
readSigned = readSigned,
readFloat = readFloat,
readString = readString,
readTerminatedString = readTerminatedString,
readSetLengthString = readSetLengthString,
readField = readField,
readBytesFast = readBytesFast,
skipStrayBits = skipStrayBits,
readUInt8 = readUInt8,
readUInt16 = readUInt16,
readUInt32 = readUInt32,
readInt8 = readInt8,
readInt16 = readInt16,
readInt32 = readInt32,
readFloat16 = readFloat16,
readFloat32 = readFloat32,
readFloat64 = readFloat64,
readBrickColor = readBrickColor,
readColor3 = readColor3,
readCFrame = readCFrame,
readVector3 = readVector3,
readVector2 = readVector2,
readUDim2 = readUDim2,
readUDim = readUDim,
readRay = readRay,
readRect = readRect,
readRegion3 = readRegion3,
readEnum = readEnum,
readNumberRange = readNumberRange,
readNumberSequence = readNumberSequence,
readColorSequence = readColorSequence,
}
end
return bitBuffer