Core I/O
Athena's primary taskflow involves use of I/O stream objects to convey data to and from a binary medium. The stream objects are used to iteratively read bytes from the data source and convert them into usable data-primitives (i.e. integers, floats, SIMD vectors).
I/O Streaming
I/O streams have a common virtual interface in Athena, so code that consumes or emits data is written against athena::io::IStreamReader and/or athena::io::IStreamWriter. Projects may use type aliasing to make Athena's namespacing more convenient for projects that use it heavily.
Assume the following example data class:
DemoRecord.hpp
#include <athena/IStreamReader.hpp>
#include <athena/IStreamWriter.hpp>
using ReadStream = athena::io::IStreamReader;
using WriteStream = athena::io::IStreamWriter;
class MyDataRecord
{
atUint32 m_valTest;
atUint32 m_valU32;
bool m_valBool;
std::string m_string;
public:
void readData(ReadStream& stream);
void writeData(WriteStream& stream) const;
};
The implementations for the I/O methods look like this:
ReadDemo.cpp
#include "DemoRecord.hpp"
void MyDataRecord::readData(ReadStream& stream)
{
/* Specify stream's default endianness as big for
* this demo. All multi-byte numeric accesses will
* adhere to this unless the 'Big' or 'Little'
* read/write method suffix is used */
stream.setEndian(athena::BigEndian);
/* A default-endian (Big here) read of 4-bytes (32-bits)
* tightly packed in the binary stream */
m_valTest = stream.readUint32();
/* Read 4-bytes and convert to 32-bit system word
* (Little to host-endian; overriding the default Big) */
m_valU32 = stream.readUint32Little();
/* Read 1-byte and convert to bool type */
m_valBool = stream.readBool();
/* Read null-terminated C-string (however long it is)
* and construct std::string to contain it */
m_string = stream.readString(-1);
}
WriteDemo.cpp
#include "DemoRecord.hpp"
void MyDataRecord::writeData(WriteStream& stream) const
{
/* Specify stream's default endianness as big for
* this demo. All multi-byte numeric accesses will
* adhere to this unless the 'Big' or 'Little'
* read/write method suffix is used */
stream.setEndian(athena::BigEndian);
/* A default-endian (Big here) write of 4-bytes (32-bits)
* tightly packed in the binary stream */
stream.writeUint32(m_valTest);
/* Emit 4-bytes with binary 32-bit word
* (host-endian to Little; overriding the default Big) */
stream.writeUint32Little(m_valU32);
/* Emit 1 byte with value 0 or 1 according to bool input */
stream.writeBool(m_valBool);
/* Write null-terminated C-string from input std::string */
stream.writeString(m_string);
}
Built-in Stream Backends
Athena ships with some useful streaming backends to get common data interfaces up and going.
File Streams (with built-in memory buffering)
ReadDemoFile.cpp
MyDataRecord ConstructAndRead(const std::string& filePath)
{
athena::io::FileReader reader(filePath);
MyDataRecord dataObject;
dataObject.readData(reader);
return dataObject;
}
WriteDemoFile.cpp
void WriteExistingObject(const std::string& filePath, const MyDataRecord& obj)
{
athena::io::FileWriter writer(filePath);
obj.writeData(writer);
}
Memory Streams (application-owned buffer)
ReadDemoMem.cpp
MyDataRecord ConstructAndRead(void* buf, atUint64 len)
{
athena::io::MemoryReader reader(buf, len);
MyDataRecord dataObject;
dataObject.readData(reader);
return dataObject;
}
WriteDemoMem.cpp
void WriteExistingObject(void* buf, atUint64 len, const MyDataRecord& obj)
{
athena::io::MemoryWriter writer(buf, len);
obj.writeData(writer);
}
Memory Copy Streams (stream-owned copy buffer)
ReadDemoMemCopy.cpp
athena::io::MemoryCopyReader ConstructAndRead(void* buf, atUint64 len, MyDataRecord& obj)
{
/* Constructor performs buffer copy */
athena::io::MemoryCopyReader reader(buf, len);
obj.readData(reader);
/* Raw data is still contained within reader for
* continued lifetime without being clobbered */
return reader;
}
WriteDemoMemCopy.cpp
athena::io::MemoryCopyWriter WriteExistingObject(void* buf, atUint64 len, const MyDataRecord& obj)
{
/* Constructor performs buffer copy */
athena::io::MemoryCopyWriter writer(buf, len);
obj.writeData(writer);
/* Raw data is still contained within writer for
* continued lifetime without being clobbered */
return writer;
}
Custom Stream Backends
The actual data interfacing occurs via raw-byte transfers using IStreamReader::readUBytesToBuf() and IStreamWriter::writeUBytes(). Programs needing to interface with custom data streams do so by inheriting IStreamReader/Writer and implementing a few methods.
MyReadBackend.hpp
#include <athena/IStreamReader.hpp>
class MyReadBackend : public athena::io::IStreamReader
{
MyReadSource& m_source
atUint64 m_curPos = 0;
public:
MyReadBackend(MyReadSource& source)
: m_source(source) {}
/* Some applications (like this demo) may not
* require random-access; leave this implementation as a stub */
void seek(atInt64 pos, SeekOrigin origin)
{}
/* Identifies current stream-position (byte index to be read next) */
atUint64 position() const
{
return m_curPos;
}
/* Identifies source length */
atUint64 length() const
{
return m_source.getLength();
}
/* Performs read to provided buffer
* (automatically called by data-primitive methods like readUint32) */
atUint64 readUBytesToBuf(void* buf, atUint64 len)
{
atUint64 rLen = m_source.readData(buf, len);
m_curPos += rLen;
return rLen;
}
};
MyWriteBackend.hpp
#include <athena/IStreamWriter.hpp>
class MyWriteBackend : public athena::io::IStreamWriter
{
MyWriteDestination& m_dest;
atUint64 m_curPos = 0;
public:
MyWriteBackend(MyWriteDestination& dest)
: m_dest(dest) {}
/* Some applications (like this demo) may not
* require random-access; leave this implementation as a stub */
void seek(atInt64 pos, SeekOrigin origin)
{}
/* Identifies current stream-position (byte index to be written next) */
atUint64 position() const
{
return m_curPos;
}
/* Identifies destination capacity */
atUint64 length() const
{
return m_dest.getCapacity();
}
/* Performs write using provided buffer
* (automatically called by data-primitive methods like writeUint32) */
atUint64 writeUBytes(void* buf, atUint64 len)
{
atUint64 wLen = m_dest.writeData(buf, len);
m_curPos += wLen;
return wLen;
}
};
DNA
Loading data field-by-field with Athena's Core has some benefits: It provides a sensible place to convert endian (byte-order) for numeric types. It can be made to handle compound types. The backing buffer is tightly-packed for compact storage (no need to worry about SIMD-alignment once read).
The major drawback is inconvenience; code needs to be written to map the data structure's fields to segments in the streamed buffer.
ATDNA: Athena's Copilot
Athena ships with a build-tool inspired by
Blender's build system
called atdna
. This tool transforms C++ record
declarations (i.e. structs, classes, unions) into reader/writer implementations
automatically. When properly integrated into a project's build system,
changes made to the C++ records will trigger these implementations to update
along with the project.
Special template types are provided by athena::io::DNA to produce a well-defined DNA Record.
DNADemo.hpp
#include <athena/DNA.hpp>
using BigDNA = athena::io::DNA<athena::BigEndian>;
struct MyDNARecord : BigDNA
{
/* This macro declares required member functions implementing the
* DNA record (generated by ATDNA and linked as a separate .cpp file) */
DECL_DNA
/* Value<T> template passes T though to the compiler and
* exposes the field to ATDNA. Primitive fields without Value<T>
* are ignored by ATDNA. */
Value<atUint32> m_val1;
Value<float> m_val2;
Value<atVec3f> m_val3;
/* Nested record declartions are also processed by ATDNA,
* assisting multi-level nested reads/writes */
struct MyDNASubRecord : BigDNA
{
DECL_DNA
Value<atUint32> m_subVal;
};
/* Vector<T,DNA_COUNT(N)> template wraps a std::vector containing
* N elements of type T. N is captured as a full C++ expression by the
* DNA_COUNT macro and pasted within the DNA record implementation. */
Value<atUint32> m_subCount;
Vector<MyDNASubRecord, DNA_COUNT(m_subCount)> m_subObjs;
};
Once the record has been defined in a header file, the header is passed
to atdna
whenever it changes. It uses
libclang
to decompose the header into C++ declarations and emits the appropriate
read/write functions according to the field types.
ATDNA + CMake
Currently, ATDNA is easiest to integrate using CMake.
Projects may define DNA targets using the atdna(<out> <in>)
macro, and
connecting the output file to a library or executable target.
CMake integrates with several build environments including make, Visual Studio, and Xcode. Please see CMake's documentation for details.
CMakeLists.txt
cmake_minimum_required(VERSION 3.0)
project(ATDNADemo)
# When Athena's codebase is built/installed on the local system,
# this package is visible from any project.
find_package(atdna REQUIRED)
# Defines the build rule to generate 'DNADemo.cpp' whenever
# 'DNADemo.hpp' changes
atdna(DNADemo.cpp DNADemo.hpp)
# Defines the executable to compile
add_executable(ATDNADemo main.cpp DNADemo.cpp DNADemo.hpp)
When built, a file like the following is generated:
DNADemo.cpp
/* Auto generated atdna implementation */
#include <athena/Global.hpp>
#include <athena/IStreamReader.hpp>
#include <athena/IStreamWriter.hpp>
#include "DNADemo.hpp"
void MyDNARecord::read(athena::io::IStreamReader& __dna_reader)
{
/* m_val1 */
m_val1 = __dna_reader.readUint32Big();
/* m_val2 */
m_val2 = __dna_reader.readFloatBig();
/* m_val3 */
m_val3 = __dna_reader.readVec3fBig();
/* m_subCount */
m_subCount = __dna_reader.readUint32Big();
/* m_subObjs */
__dna_reader.enumerate(m_subObjs, m_subCount);
}
void MyDNARecord::write(athena::io::IStreamWriter& __dna_writer) const
{
/* m_val1 */
__dna_writer.writeUint32Big(m_val1);
/* m_val2 */
__dna_writer.writeFloatBig(m_val2);
/* m_val3 */
__dna_writer.writeVec3fBig(m_val3);
/* m_subCount */
__dna_writer.writeUint32Big(m_subCount);
/* m_subObjs */
__dna_writer.enumerate(m_subObjs);
}
void MyDNARecord::MyDNASubRecord::read(athena::io::IStreamReader& __dna_reader)
{
/* m_subVal */
m_subVal = __dna_reader.readUint32Big();
}
void MyDNARecord::MyDNASubRecord::write(athena::io::IStreamWriter& __dna_writer) const
{
/* m_subVal */
__dna_writer.writeUint32Big(m_subVal);
}
All together now!
Once the read/write implementations are compiled in, the application may invoke them however's convenient:
main.cpp
#include <iostream>
#include <athena/FileReader.hpp>
#include "DNADemo.hpp"
int main(int argc, char* argv[])
{
athena::io::FileReader reader("MyDemoData.bin");
MyDNARecord record;
record.read(reader); /* DNA implementation called here */
std::cout << "Val1: " << record.m_val1 << " Val2: " << record.m_val2 << "\n";
return 0;
}
main.cpp
#include <iostream>
#include <athena/FileWriter.hpp>
#include "DNADemo.cpp"
int main(int argc, char* argv[])
{
athena::io::FileWriter writer("MyDemoData.bin");
MyDNARecord record;
record.m_val1 = 0x42;
record.m_val2 = 3.14159265359;
record.write(writer); /* DNA implementation called here */
return 0;
}
YAML
Having a uniform system to interchange binary data is nice, but we musn't forget the human programmers that work with the data and design systems around it. This is where having a textual representation of the structured data is convenient. YAML is a simplistic data-serialization format, capable of organizing string-representations of data members into mappings and sequences with multiple levels of hierarchy.
athena::io::DNA has been subclassed as athena::io::DNAYaml to have ATDNA generate YAML serialization/deserialization alongside the binary readers/writers. It's used just like the DNA system from the developer's perspective.
YAMLDemo.hpp
#include <athena/DNAYaml.hpp>
using BigYAML = athena::io::DNAYaml<athena::BigEndian>;
struct MyYAMLRecord : BigYAML
{
/* This macro declares required member functions implementing the
* YAML record (generated by ATDNA and linked as a separate .cpp file) */
DECL_YAML
/* Value<T> template passes T though to the compiler and
* exposes the field to ATDNA. Primitive fields without Value<T>
* are ignored by ATDNA. */
Value<atUint32> m_val1;
Value<float> m_val2;
Value<atVec3f> m_val3;
/* Nested record declartions are also processed by ATDNA,
* assisting multi-level nested reads/writes */
struct MyYAMLSubRecord : BigYAML
{
DECL_YAML
Value<atUint32> m_subVal;
};
/* Vector<T,DNA_COUNT(N)> template wraps a std::vector containing
* N elements of type T. N is captured as a full C++ expression by the
* DNA_COUNT macro and pasted within the YAML record implementation. */
Value<atUint32> m_subCount;
Vector<MyYAMLSubRecord, DNA_COUNT(m_subCount)> m_subObjs;
};
Now applications can use YAML as a data source/destination in addition to the original binary format the DNA is based on. Such YAML may look like this:
YAMLDemo.yaml
m_val1: 0x42
m_val2: 3.14159265359
m_val3: [1.000000, 2.000000, 3.000000]
m_subCount: 0x3
m_subObjs:
- {m_subVal: 0x1}
- {m_subVal: 0x2}
- {m_subVal: 0x3}
All together now!
The YAML implementations are compiled side-by-side with the DNA implementations. The application may invoke them in a similar manner:
main.cpp
#include <stdio.h>
#include <iostream>
#include "YAMLDemo.hpp"
int main(int argc, char* argv[])
{
/* Stdio FILEs are one option. Raw string buffers are also available */
FILE* fp = fopen("MyDemoYAML.yaml", "r");
MyYAMLRecord record;
record.fromYAMLFile(fp); /* YAML implementation called here */
std::cout << "Val1: " << record.m_val1 << " Val2: " << record.m_val2 << "\n";
fclose(fp);
return 0;
}
main.cpp
#include <stdio.h>
#include <iostream>
#include "YAMLDemo.hpp"
int main(int argc, char* argv[])
{
/* Stdio FILEs are one option. Raw string buffers are also available */
FILE* fp = fopen("MyDemoYAML.yaml", "w");
MyYAMLRecord record;
record.m_val1 = 0x42;
record.m_val2 = 3.14159265359;
record.toYAMLFile(fp); /* YAML implementation called here */
fclose(fp);
return 0;
}