Tables.jl

When consuming a table, you can attempt to retrieve its schema using Tables.schema(rows).

If the table source is exotic (e.g., a lazy transformation) and Tables.schema returns nothing, you can fallback to an 'unknown schema' strategy:

1. Start iterating the rows using iterate(rows).
2. Use the first available row to extract column names via Tables.columnnames(row).
3. Construct a partial Tables.Schema(names, nothing) to allow subsequent processing (like Tables.eachcolumn) to function correctly.

sch = Tables.schema(rows)
if sch === nothing
    # Fallback for unknown schema
    state = iterate(rows)
    state === nothing && return
    row, st = state
    
    names = Tables.columnnames(row)
    sch = Tables.Schema(names, nothing)
    
    # Continue iteration using the state from the first row
    while true
        Tables.eachcolumn(sch, row) do val, i, nm
            # process values
        end
        state = iterate(rows, st)
        state === nothing && break
        row, st = state
    end
end

To make a custom type MyTable compatible with Tables.jl consumers, you must implement a specific set of required methods. You must choose between a row-oriented or column-oriented implementation.

1. Declare the type as a table

You must first signal that your type implements the interface:

• Tables.istable(::Type{MyTable}) = true

2. Choose an access pattern

You must implement one of the following two patterns:

Row-oriented (Access via Tables.rows)

Use this if your type naturally iterates over rows.

• Tables.rowaccess(::Type{MyTable}) = true
• Tables.rows(x::MyTable): Returns an Tables.AbstractRow-compatible iterator.

Column-oriented (Access via Tables.columns)

Use this if your type naturally provides access to columns.

• Tables.columnaccess(::Type{MyTable}) = true
• Tables.columns(x::MyTable): Returns an Tables.AbstractColumns-compatible object.

3. Optional methods for optimization

• Tables.schema(x::MyTable): Returns a Tables.Schema object (column names and types) or nothing.
• Tables.materializer(::Type{MyTable}): A sink function to construct your type from Tables.jl input.
• Tables.subset(x::MyTable, inds; viewhint): Returns a row or sub-table.
• DataAPI.nrow(x::MyTable): Returns the number of rows.
• DataAPI.ncol(x::MyTable): Returns the number of columns.

# Example: Minimal Row-oriented setup
struct MyTable
    # ... fields
end

Tables.istable(::Type{MyTable}) = true
Tables.rowaccess(::Type{MyTable}) = true
Tables.rows(x::MyTable) = # return an iterator of AbstractRow

To ensure your implementation is robust, perform "round-trip" testing by converting between different table formats and plain Julia Matrix objects using Tables.matrix.

1. Source to Sink: Convert a table (like a Vector of NamedTuples or a NamedTuple of Vectors) into a Matrix using Tables.matrix(table).
2. Sink to Source: Convert a Matrix back into a table using Tables.table(matrix) and then use rowtable or columntable to transform it into specific row-oriented or column-oriented formats.

Note that Vector of NamedTuples acts as a default row table, and NamedTuple of Vectors acts as a default column table.

# Define test data
rt = [(a=1, b=4.0, c="7"), (a=2, b=5.0, c="8"), (a=3, b=6.0, c="9")] # Row table
ct = (a=[1,2,3], b=[4.0, 5.0, 6.0])                                   # Column table
mat_input = [1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"]

# 1. Test consuming a row table
mat = Tables.matrix(rt)
@test mat[:, 1] == [1, 2, 3]
@test size(mat) == (3, 3)

# 2. Test consuming a column table
mat2 = Tables.matrix(ct)
@test mat2[:, 1] == ct.a

# 3. Test round-trip: Matrix -> Column Table
tbl = Tables.table(mat_input) |> columntable
@test keys(tbl) == (:Column1, :Column2, :Column3)

# 4. Test round-trip: Matrix -> Row Table
tbl2 = Tables.table(mat2) |> rowtable
@test length(tbl2) == 3
@test map(x->x.Column1, tbl2) == [1.0, 2.0, 3.0]

When implementing a new table source, you should verify that your type correctly responds to the Tables.jl interface. Key checks include:

1. Identity and Access: Use Tables.istable(typeof(your_type)) to confirm it is recognized as a table, and check Tables.rowaccess and Tables.columnaccess to see which interface is supported.
2. Row/Column Access: Verify that Tables.rows(your_table) and Tables.columns(your_table) return the expected objects.
3. Column Interface: Test Tables.getcolumn(table, name_or_index), Tables.columnnames(table), and ensure property access (e.g., table.ColumnName) works if applicable.
4. Row Interface: Iterate through rows and verify that eltype(rows) matches the type of the row object, and that Tables.getcolumn(row, name_or_index) works correctly on the row itself.
5. Property Consistency: Ensure propertynames(table) matches propertynames(row).

mat = [1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"]
# first, create a MatrixTable from our matrix input
mattbl = Tables.table(mat)

# test that the MatrixTable `istable`
@test Tables.istable(typeof(mattbl))
# test that it defines row access
@test Tables.rowaccess(typeof(mattbl))
matrows = Tables.rows(mattbl)
@test Tables.rows(matrows) === matrows
# test that it defines column access
@test Tables.columnaccess(typeof(mattbl))
@test Tables.columns(mattbl) === mattbl
# test that we can access the first "column" of our matrix table by column name
@test mattbl.Column1 == [1,2,3]
# test our `Tables.AbstractColumns` interface methods
@test Tables.getcolumn(mattbl, :Column1) == [1,2,3]
@test Tables.getcolumn(mattbl, 1) == [1,2,3]
@test Tables.columnnames(mattbl) == [:Column1, :Column2, :Column3]
# now let's iterate our MatrixRowTable to get our first MatrixRow
matrow = first(matrows)
@test eltype(matrows) == typeof(matrow)
# now we can test our `Tables.AbstractRow` interface methods on our MatrixRow
@test matrow.Column1 == 1
@test Tables.getcolumn(matrow, :Column1) == 1
@test Tables.getcolumn(matrow, 1) == 1
@test propertynames(mattbl) == propertynames(matrow) == [:Column1, :Column2, :Column3]

This example demonstrates how to wrap an AbstractVecOrMat to make it a valid Tables.jl source by implementing both row and column interfaces.

struct MatrixTable{T <: AbstractVecOrMat} <: Tables.AbstractColumns
    names::Vector{Symbol}
    lookup::Dict{Symbol, Int}
    matrix::T
end

# 1. Table Declaration
Tables.istable(::Type{<:MatrixTable}) = true

# 2. Schema
Tables.schema(m::MatrixTable{T}) where {T} = Tables.Schema(names(m), fill(eltype(T), size(matrix(m), 2)))

# 3. Column Interface
Tables.columnaccess(::Type{<:MatrixTable}) = true
Tables.columns(m::MatrixTable) = m
Tables.getcolumn(m::MatrixTable, ::Type{T}, col::Int, nm::Symbol) where {T} = matrix(m)[:, col]
Tables.getcolumn(m::MatrixTable, nm::Symbol) = matrix(m)[:, lookup(m)[nm]]
Tables.getcolumn(m::MatrixTable, i::Int) = matrix(m)[:, i]
Tables.columnnames(m::MatrixTable) = names(m)

# 4. Row Interface
Tables.rowaccess(::Type{<:MatrixTable}) = true
Tables.rows(m::MatrixTable) = m

# Iteration Interface
Base.eltype(m::MatrixTable{T}) where {T} = MatrixRow{T}
Base.length(m::MatrixTable) = size(matrix(m), 1)
Base.iterate(m::MatrixTable, st=1) = st > length(m) ? nothing : (MatrixRow(st, m), st + 1)

struct MatrixRow{T} <: Tables.AbstractRow
    row::Int
    source::MatrixTable{T}
end

getcolumn(m::MatrixRow, ::Type, col::Int, nm::Symbol) = getfield(getfield(m, :source), :matrix)[getfield(m, :row), col]
getcolumn(m::MatrixRow, i::Int) = getfield(getfield(m, :source), :matrix)[getfield(m, :row), i]
getcolumn(m::MatrixRow, nm::Symbol) = getfield(getfield(m, :source), :matrix)[getfield(m, :row), getfield(getfield(m, :source), :lookup)[nm]]
columnnames(m::MatrixRow) = names(getfield(m, :source))