Core I/O

Athena's primary taskflow involves use of I/O stream objects to convey data to and from a binary medium. The stream objects are used to iteratively read bytes from the data source and convert them into usable data-primitives (i.e. integers, floats, SIMD vectors).

I/O Streaming

I/O streams have a common virtual interface in Athena, so code that consumes or emits data is written against athena::io::IStreamReader and/or athena::io::IStreamWriter. Projects may use type aliasing to make Athena's namespacing more convenient for projects that use it heavily.

Assume the following example data class:

DemoRecord.hpp

#include <athena/IStreamReader.hpp>
#include <athena/IStreamWriter.hpp>
using ReadStream = athena::io::IStreamReader;
using WriteStream = athena::io::IStreamWriter;

class MyDataRecord
{
    atUint32 m_valTest;
    atUint32 m_valU32;
    bool m_valBool;
    std::string m_string;
public:
    void readData(ReadStream& stream);
    void writeData(WriteStream& stream) const;
};

The implementations for the I/O methods look like this:

ReadDemo.cpp

#include "DemoRecord.hpp"
void MyDataRecord::readData(ReadStream& stream)
{
    /* Specify stream's default endianness as big for 
     * this demo. All multi-byte numeric accesses will
     * adhere to this unless the 'Big' or 'Little' 
     * read/write method suffix is used */
    stream.setEndian(athena::BigEndian);

    /* A default-endian (Big here) read of 4-bytes (32-bits)
     * tightly packed in the binary stream */
    m_valTest = stream.readUint32();

    /* Read 4-bytes and convert to 32-bit system word 
     * (Little to host-endian; overriding the default Big) */
    m_valU32 = stream.readUint32Little();

    /* Read 1-byte and convert to bool type */
    m_valBool = stream.readBool();

    /* Read null-terminated C-string (however long it is) 
     * and construct std::string to contain it */
    m_string = stream.readString(-1);
}

WriteDemo.cpp

#include "DemoRecord.hpp"
void MyDataRecord::writeData(WriteStream& stream) const
{
    /* Specify stream's default endianness as big for 
     * this demo. All multi-byte numeric accesses will
     * adhere to this unless the 'Big' or 'Little' 
     * read/write method suffix is used */
    stream.setEndian(athena::BigEndian);

    /* A default-endian (Big here) write of 4-bytes (32-bits)
     * tightly packed in the binary stream */
    stream.writeUint32(m_valTest);

    /* Emit 4-bytes with binary 32-bit word 
     * (host-endian to Little; overriding the default Big) */
    stream.writeUint32Little(m_valU32);

    /* Emit 1 byte with value 0 or 1 according to bool input */
    stream.writeBool(m_valBool);

    /* Write null-terminated C-string from input std::string */
    stream.writeString(m_string);
}

Built-in Stream Backends

Athena ships with some useful streaming backends to get common data interfaces up and going.

File Streams (with built-in memory buffering)

ReadDemoFile.cpp

MyDataRecord ConstructAndRead(const std::string& filePath)
{
    athena::io::FileReader reader(filePath);
    MyDataRecord dataObject;
    dataObject.readData(reader);
    return dataObject;
}

WriteDemoFile.cpp

void WriteExistingObject(const std::string& filePath, const MyDataRecord& obj)
{
    athena::io::FileWriter writer(filePath);
    obj.writeData(writer);
}

Memory Streams (application-owned buffer)

ReadDemoMem.cpp

MyDataRecord ConstructAndRead(void* buf, atUint64 len)
{
    athena::io::MemoryReader reader(buf, len);
    MyDataRecord dataObject;
    dataObject.readData(reader);
    return dataObject;
}

WriteDemoMem.cpp

void WriteExistingObject(void* buf, atUint64 len, const MyDataRecord& obj)
{
    athena::io::MemoryWriter writer(buf, len);
    obj.writeData(writer);
}

Memory Copy Streams (stream-owned copy buffer)

ReadDemoMemCopy.cpp

athena::io::MemoryCopyReader ConstructAndRead(void* buf, atUint64 len, MyDataRecord& obj)
{
    /* Constructor performs buffer copy */
    athena::io::MemoryCopyReader reader(buf, len);
    obj.readData(reader);

    /* Raw data is still contained within reader for 
     * continued lifetime without being clobbered */
    return reader; 
}

WriteDemoMemCopy.cpp

athena::io::MemoryCopyWriter WriteExistingObject(void* buf, atUint64 len, const MyDataRecord& obj)
{
    /* Constructor performs buffer copy */
    athena::io::MemoryCopyWriter writer(buf, len);
    obj.writeData(writer);

    /* Raw data is still contained within writer for 
     * continued lifetime without being clobbered */
    return writer;
}

Custom Stream Backends

The actual data interfacing occurs via raw-byte transfers using IStreamReader::readUBytesToBuf() and IStreamWriter::writeUBytes(). Programs needing to interface with custom data streams do so by inheriting IStreamReader/Writer and implementing a few methods.

MyReadBackend.hpp

#include <athena/IStreamReader.hpp>
class MyReadBackend : public athena::io::IStreamReader
{
    MyReadSource& m_source
    atUint64 m_curPos = 0;
public:
    MyReadBackend(MyReadSource& source)
    : m_source(source) {}

    /* Some applications (like this demo) may not
     * require random-access; leave this implementation as a stub */
    void seek(atInt64 pos, SeekOrigin origin)
    {}

    /* Identifies current stream-position (byte index to be read next) */
    atUint64 position() const
    {
        return m_curPos;
    }

    /* Identifies source length */
    atUint64 length() const
    {
        return m_source.getLength();
    }

    /* Performs read to provided buffer 
     * (automatically called by data-primitive methods like readUint32) */
    atUint64 readUBytesToBuf(void* buf, atUint64 len)
    {
        atUint64 rLen = m_source.readData(buf, len);
        m_curPos += rLen;
        return rLen;
    }
};

MyWriteBackend.hpp

#include <athena/IStreamWriter.hpp>
class MyWriteBackend : public athena::io::IStreamWriter
{
    MyWriteDestination& m_dest;
    atUint64 m_curPos = 0;
public:
    MyWriteBackend(MyWriteDestination& dest)
    : m_dest(dest) {}

    /* Some applications (like this demo) may not
     * require random-access; leave this implementation as a stub */
    void seek(atInt64 pos, SeekOrigin origin)
    {}

    /* Identifies current stream-position (byte index to be written next) */
    atUint64 position() const
    {
        return m_curPos;
    }

    /* Identifies destination capacity */
    atUint64 length() const
    {
        return m_dest.getCapacity();
    }

    /* Performs write using provided buffer 
     * (automatically called by data-primitive methods like writeUint32) */
    atUint64 writeUBytes(void* buf, atUint64 len)
    {
        atUint64 wLen = m_dest.writeData(buf, len);
        m_curPos += wLen;
        return wLen;
    }
};

DNA

Loading data field-by-field with Athena's Core has some benefits: It provides a sensible place to convert endian (byte-order) for numeric types. It can be made to handle compound types. The backing buffer is tightly-packed for compact storage (no need to worry about SIMD-alignment once read).

The major drawback is inconvenience; code needs to be written to map the data structure's fields to segments in the streamed buffer.

ATDNA: Athena's Copilot

Athena ships with a build-tool inspired by Blender's build system called atdna. This tool transforms C++ record declarations (i.e. structs, classes, unions) into reader/writer implementations automatically. When properly integrated into a project's build system, changes made to the C++ records will trigger these implementations to update along with the project.

Special template types are provided by athena::io::DNA to produce a well-defined DNA Record.

DNADemo.hpp

#include <athena/DNA.hpp>
using BigDNA = athena::io::DNA<athena::BigEndian>;

struct MyDNARecord : BigDNA
{
    /* This macro declares required member functions implementing the 
     * DNA record (generated by ATDNA and linked as a separate .cpp file) */
    DECL_DNA

    /* Value<T> template passes T though to the compiler and 
     * exposes the field to ATDNA. Primitive fields without Value<T>
     * are ignored by ATDNA. */
    Value<atUint32> m_val1;
    Value<float> m_val2;
    Value<atVec3f> m_val3;

    /* Nested record declartions are also processed by ATDNA, 
     * assisting multi-level nested reads/writes */
    struct MyDNASubRecord : BigDNA
    {
        DECL_DNA
        Value<atUint32> m_subVal;
    };

    /* Vector<T,DNA_COUNT(N)> template wraps a std::vector containing 
     * N elements of type T. N is captured as a full C++ expression by the 
     * DNA_COUNT macro and pasted within the DNA record implementation. */
    Value<atUint32> m_subCount;
    Vector<MyDNASubRecord, DNA_COUNT(m_subCount)> m_subObjs;
};

Once the record has been defined in a header file, the header is passed to atdna whenever it changes. It uses libclang to decompose the header into C++ declarations and emits the appropriate read/write functions according to the field types.

ATDNA + CMake

Currently, ATDNA is easiest to integrate using CMake. Projects may define DNA targets using the atdna(<out> <in>) macro, and connecting the output file to a library or executable target.

CMake integrates with several build environments including make, Visual Studio, and Xcode. Please see CMake's documentation for details.

CMakeLists.txt

cmake_minimum_required(VERSION 3.0)
project(ATDNADemo)

# When Athena's codebase is built/installed on the local system,
# this package is visible from any project.
find_package(atdna REQUIRED)

# Defines the build rule to generate 'DNADemo.cpp' whenever 
# 'DNADemo.hpp' changes
atdna(DNADemo.cpp DNADemo.hpp)

# Defines the executable to compile
add_executable(ATDNADemo main.cpp DNADemo.cpp DNADemo.hpp)

When built, a file like the following is generated:

DNADemo.cpp

/* Auto generated atdna implementation */
#include <athena/Global.hpp>
#include <athena/IStreamReader.hpp>
#include <athena/IStreamWriter.hpp>

#include "DNADemo.hpp"

void MyDNARecord::read(athena::io::IStreamReader& __dna_reader)
{
    /* m_val1 */
    m_val1 = __dna_reader.readUint32Big();
    /* m_val2 */
    m_val2 = __dna_reader.readFloatBig();
    /* m_val3 */
    m_val3 = __dna_reader.readVec3fBig();
    /* m_subCount */
    m_subCount = __dna_reader.readUint32Big();
    /* m_subObjs */
    __dna_reader.enumerate(m_subObjs, m_subCount);
}

void MyDNARecord::write(athena::io::IStreamWriter& __dna_writer) const
{
    /* m_val1 */
    __dna_writer.writeUint32Big(m_val1);
    /* m_val2 */
    __dna_writer.writeFloatBig(m_val2);
    /* m_val3 */
    __dna_writer.writeVec3fBig(m_val3);
    /* m_subCount */
    __dna_writer.writeUint32Big(m_subCount);
    /* m_subObjs */
    __dna_writer.enumerate(m_subObjs);
}

void MyDNARecord::MyDNASubRecord::read(athena::io::IStreamReader& __dna_reader)
{
    /* m_subVal */
    m_subVal = __dna_reader.readUint32Big();
}

void MyDNARecord::MyDNASubRecord::write(athena::io::IStreamWriter& __dna_writer) const
{
    /* m_subVal */
    __dna_writer.writeUint32Big(m_subVal);
}

All together now!

Once the read/write implementations are compiled in, the application may invoke them however's convenient:

main.cpp

#include <iostream>
#include <athena/FileReader.hpp>
#include "DNADemo.hpp"
int main(int argc, char* argv[])
{
    athena::io::FileReader reader("MyDemoData.bin");
    MyDNARecord record;
    record.read(reader); /* DNA implementation called here */
    std::cout << "Val1: " << record.m_val1 << " Val2: " << record.m_val2 << "\n";
    return 0;
}

main.cpp

#include <iostream>
#include <athena/FileWriter.hpp>
#include "DNADemo.cpp"
int main(int argc, char* argv[])
{
    athena::io::FileWriter writer("MyDemoData.bin");
    MyDNARecord record;
    record.m_val1 = 0x42;
    record.m_val2 = 3.14159265359;
    record.write(writer); /* DNA implementation called here */
    return 0;
}

YAML

Having a uniform system to interchange binary data is nice, but we musn't forget the human programmers that work with the data and design systems around it. This is where having a textual representation of the structured data is convenient. YAML is a simplistic data-serialization format, capable of organizing string-representations of data members into mappings and sequences with multiple levels of hierarchy.

athena::io::DNA has been subclassed as athena::io::DNAYaml to have ATDNA generate YAML serialization/deserialization alongside the binary readers/writers. It's used just like the DNA system from the developer's perspective.

YAMLDemo.hpp

#include <athena/DNAYaml.hpp>
using BigYAML = athena::io::DNAYaml<athena::BigEndian>;

struct MyYAMLRecord : BigYAML
{
    /* This macro declares required member functions implementing the 
     * YAML record (generated by ATDNA and linked as a separate .cpp file) */
    DECL_YAML

    /* Value<T> template passes T though to the compiler and 
     * exposes the field to ATDNA. Primitive fields without Value<T>
     * are ignored by ATDNA. */
    Value<atUint32> m_val1;
    Value<float> m_val2;
    Value<atVec3f> m_val3;

    /* Nested record declartions are also processed by ATDNA, 
     * assisting multi-level nested reads/writes */
    struct MyYAMLSubRecord : BigYAML
    {
        DECL_YAML
        Value<atUint32> m_subVal;
    };

    /* Vector<T,DNA_COUNT(N)> template wraps a std::vector containing 
     * N elements of type T. N is captured as a full C++ expression by the 
     * DNA_COUNT macro and pasted within the YAML record implementation. */
    Value<atUint32> m_subCount;
    Vector<MyYAMLSubRecord, DNA_COUNT(m_subCount)> m_subObjs;
};

Now applications can use YAML as a data source/destination in addition to the original binary format the DNA is based on. Such YAML may look like this:

YAMLDemo.yaml

m_val1: 0x42
m_val2: 3.14159265359
m_val3: [1.000000, 2.000000, 3.000000]
m_subCount: 0x3
m_subObjs:
- {m_subVal: 0x1}
- {m_subVal: 0x2}
- {m_subVal: 0x3}

All together now!

The YAML implementations are compiled side-by-side with the DNA implementations. The application may invoke them in a similar manner:

main.cpp

#include <stdio.h>
#include <iostream>
#include "YAMLDemo.hpp"
int main(int argc, char* argv[])
{
    /* Stdio FILEs are one option. Raw string buffers are also available */
    FILE* fp = fopen("MyDemoYAML.yaml", "r");
    MyYAMLRecord record;
    record.fromYAMLFile(fp); /* YAML implementation called here */
    std::cout << "Val1: " << record.m_val1 << " Val2: " << record.m_val2 << "\n";
    fclose(fp);
    return 0;
}

main.cpp

#include <stdio.h>
#include <iostream>
#include "YAMLDemo.hpp"
int main(int argc, char* argv[])
{
    /* Stdio FILEs are one option. Raw string buffers are also available */
    FILE* fp = fopen("MyDemoYAML.yaml", "w");
    MyYAMLRecord record;
    record.m_val1 = 0x42;
    record.m_val2 = 3.14159265359;
    record.toYAMLFile(fp); /* YAML implementation called here */
    fclose(fp);
    return 0;
}