port to Julia 0.7

.travis.yml

@@ -4,7 +4,7 @@ os:
   - linux
   - osx
 julia:
-  - 0.6
+  - nightly
 notifications:
   email: false
 # uncomment the following lines to override the default test script

REQUIRE

@@ -1,4 +1,4 @@
-julia 0.6
+julia 0.7-
 Compat 0.19
 NamedTuples 2.1.0
 OnlineStats 0.17

src/IndexedTables.jl

@@ -2,7 +2,7 @@ __precompile__()
 module IndexedTables
 
 using Compat
-using NamedTuples, PooledArrays
+using PooledArrays
 
 import Base:
     show, eltype, length, getindex, setindex!, ndims, map, convert, keys, values,
@@ -58,6 +58,6 @@ include("join.jl")
 include("reshape.jl")
 
 # TableTraits.jl integration
-include("tabletraits.jl")
+#include("tabletraits.jl")
 
 end # module

src/collect.jl

@@ -1,13 +1,7 @@
-import DataValues: DataValue
-
 _is_subtype(::Type{S}, ::Type{T}) where {S, T} = promote_type(S, T) == T
 
 Base.@pure function dataarrayof(T)
-    if T<:DataValue
-        DataValueArray{T.parameters[1],1}
-    else
-        Vector{T}
-    end
+    Vector{T}
 end
 
 """
@@ -39,7 +33,7 @@ julia> collect_columns(s)
 collect_columns(itr) = collect_columns(itr, Base.iteratorsize(itr))
 
 function collect_empty_columns(itr::T) where {T}
-    S = Core.Inference.return_type(first, Tuple{T})
+    S = Core.Compiler.return_type(first, Tuple{T})
     similar(arrayof(S), 0)
 end
 
@@ -147,7 +141,7 @@ fieldwise_isa(el::S, ::Type{Tuple}) where {S<:Tup} = _is_subtype(S, Tuple)
 fieldwise_isa(el::S, ::Type{NamedTuple}) where {S<:Tup} = _is_subtype(S, NamedTuple)
 
 @generated function fieldwise_isa(el::S, ::Type{T}) where {S<:Tup, T<:Tup}
-    if (fieldnames(S) == fieldnames(T)) && all(_is_subtype(s, t) for (s, t) in zip(S.parameters, T.parameters))
+    if (fieldnames(S) == fieldnames(T)) && all(_is_subtype(s, t) for (s, t) in zip(fieldtypes(S), fieldtypes(T)))
         return :(true)
     else
         return :(false)
@@ -171,8 +165,8 @@ function widencolumns(dest, i, el::S, ::Type{T}) where{S <: Tup, T<:Tup}
         new = Array{R}(length(dest))
         copy!(new, 1, dest, 1, i-1)
     else
-        sp, tp = S.parameters, T.parameters
-        idx = find(!(s <: t) for (s, t) in zip(sp, tp))
+        sp, tp = fieldtypes(S), fieldtypes(T)
+        idx = findall(collect(!(s <: t) for (s, t) in zip(sp, tp)))
         new = dest
         for l in idx
             newcol = dataarrayof(promote_type(sp[l], tp[l]))(length(dest))

src/columns.jl

@@ -2,12 +2,14 @@ using Compat
 
 import Base:
     linearindexing, push!, size, sort, sort!, permute!, issorted, sortperm,
-    summary, resize!, vcat, serialize, deserialize, append!, copy!, view
+    summary, resize!, vcat, append!, copy!, view
 
 export Columns, colnames, ncols, ColDict, insertafter!, insertbefore!, @cols, setcol, pushcol, popcol, insertcol, insertcolafter, insertcolbefore, renamecol
 export map_rows
 export All, Not, Between, Keys
 
+import Serialization: serialize, deserialize
+
 """
 A type that stores an array of tuples as a tuple of arrays.
 
@@ -34,20 +36,25 @@ struct Columns{D<:Union{Tup, Pair}, C<:Union{Tup, Pair}} <: AbstractVector{D}
     end
 end
 
-function Columns(cols::AbstractVector...; names::Union{Vector,Tuple{Vararg{Any}},Void}=nothing)
-    if isa(names, Void) || any(x->!(x isa Symbol), names)
+function Columns(cols::AbstractVector...; names::Union{Vector,Tuple{Vararg{Any}},Nothing}=nothing)
+    if isa(names, Nothing) || any(x->!(x isa Symbol), names)
         Columns{eltypes(typeof(cols)),typeof(cols)}(cols)
     else
-        dt = eval(:(@NT($(names...)))){map(eltype, cols)...}
-        ct = eval(:(@NT($(names...)))){map(typeof, cols)...}
-        Columns{dt,ct}(ct(cols...))
+        dt = NamedTuple{(names...,), Tuple{map(eltype, cols)...}}
+        ct = NamedTuple{(names...,), Tuple{map(typeof, cols)...}}
+        Columns{dt,ct}(ct((cols...,)))
     end
 end
 
-Columns(; pairs...) = Columns(map(x->x[2],pairs)..., names=Symbol[x[1] for x in pairs])
+function Columns(; kws...)
+    Columns(values(kws)..., names=collect(keys(kws)))
+end
 
 Columns(c::Union{Tup, Pair}) = Columns{eltypes(typeof(c)),typeof(c)}(c)
 
+# There is a StackOverflow bug in this case in Base.unaliascopy
+Base.copy(c::Columns{<:Union{NamedTuple{(),Tuple{}}, Tuple{}}}) = c
+
 # IndexedTable-like API
 
 """
@@ -113,7 +120,7 @@ julia> colnames(ndsparse(Columns(x=[1,2,3]), Columns([3,4,5],[6,7,8])))
 """
 function colnames end
 
-Base.@pure colnames(t::AbstractVector) = [1]
+Base.@pure colnames(t::AbstractVector) = (1,)
 columns(v::AbstractVector) = v
 
 Base.@pure colnames(t::Columns) = fieldnames(eltype(t))
@@ -211,25 +218,28 @@ function similar(c::Columns{D,C}, n::Integer) where {D,C}
     Columns{D,typeof(cols)}(cols)
 end
 
-function Base.similar{T<:Columns}(::Type{T}, n::Int)::T
+function Base.similar(::Type{T}, n::Int)::T where {T<:Columns}
     T_cols = T.parameters[2]
-    f = T_cols <: Tuple ? tuple : T_cols
-    T(f(map(t->similar(t, n), T.parameters[2].parameters)...))
+    if T_cols <: Pair
+        return Columns(similar(T_cols.parameters[1], n) => similar(T_cols.parameters[2], n))
+    end
+    f = T_cols <: Tuple ? tuple : T_cols∘tuple
+    T(f(map(t->similar(t, n), fieldtypes(T_cols))...))
 end
 
 function convert(::Type{Columns}, x::AbstractArray{<:NTuple{N,Any}}) where N
     eltypes = (eltype(x).parameters...)
     copy!(Columns(map(t->Vector{t}(length(x)), eltypes)), x)
 end
 
-function convert(::Type{Columns}, x::AbstractArray{<:NamedTuple})
-    eltypes = (eltype(x).parameters...)
+function convert(::Type{Columns}, x::AbstractArray{<:NamedTuple{names, typs}}) where {names,typs}
+    eltypes = typs.parameters
     copy!(Columns(map(t->Vector{t}(length(x)), eltypes)..., names=fieldnames(eltype(x))), x)
 end
 
 
 getindex(c::Columns{D}, i::Integer) where {D<:Tuple} = ith_all(i, c.columns)
-getindex(c::Columns{D}, i::Integer) where {D<:NamedTuple} = D(ith_all(i, c.columns)...)
+getindex(c::Columns{D}, i::Integer) where {D<:NamedTuple} = D(ith_all(i, c.columns))
 getindex(c::Columns{D}, i::Integer) where {D<:Pair} = getindex(c.columns.first, i) => getindex(c.columns.second, i)
 
 getindex(c::Columns, p::AbstractVector) = Columns(_map(c->c[p], c.columns))
@@ -398,7 +408,7 @@ pushrow!(to::Columns, from::Columns, i) = foreach((a,b)->push!(a, b[i]), to.colu
 pushrow!(to::AbstractArray, from::AbstractArray, i) = push!(to, from[i])
 
 @generated function rowless(c::Columns{D,C}, i, j) where {D,C}
-    N = length(C.parameters)
+    N = nfields(C)
     ex = :(cmpelts(getfield(c.columns,$N), i, j) < 0)
     for n in N-1:-1:1
         ex = quote
@@ -411,7 +421,7 @@ pushrow!(to::AbstractArray, from::AbstractArray, i) = push!(to, from[i])
 end
 
 @generated function roweq(c::Columns{D,C}, i, j) where {D,C}
-    N = length(C.parameters)
+    N = nfields(C)
     ex = :(cmpelts(getfield(c.columns,1), i, j) == 0)
     for n in 2:N
         ex = :(($ex) && (cmpelts(getfield(c.columns,$n), i, j)==0))
@@ -424,7 +434,7 @@ end
 # uses number of columns from `d`, assuming `c` has more or equal
 # dimensions, for broadcast joins.
 @generated function rowcmp(c::Columns, i, d::Columns{D}, j) where D
-    N = length(D.parameters)
+    N = nfields(D)
     ex = :(cmp(getfield(c.columns,$N)[i], getfield(d.columns,$N)[j]))
     for n in N-1:-1:1
         ex = quote
@@ -657,7 +667,7 @@ lowerselection(t, s::Tuple)              = map(x -> lowerselection(t, x), s)
 lowerselection(t, s::Not)                = excludecols(t, lowerselection(t, s.cols))
 lowerselection(t, s::Keys)               = lowerselection(t, IndexedTables.pkeynames(t))
 lowerselection(t, s::Between)            = Tuple(colindex(t, s.first):colindex(t, s.last))
-lowerselection(t, s::Function)           = colindex(t, Tuple(filter(s, colnames(t))))
+lowerselection(t, s::Function)           = colindex(t, Tuple(filter(s, collect(colnames(t)))))
 lowerselection(t, s::Regex)              = lowerselection(t, x -> ismatch(s, string(x)))
 
 function lowerselection(t, s::All)
@@ -684,7 +694,7 @@ function colindex(t, col::SpecialSelector)
     colindex(t, lowerselection(t, col))
 end
 
-function _colindex(fnames::AbstractArray, col, default=nothing)
+function _colindex(fnames::Union{Tuple, AbstractArray}, col, default=nothing)
     if isa(col, Int) && 1 <= col <= length(fnames)
         return col
     elseif isa(col, Symbol)
@@ -722,7 +732,7 @@ function columns(c, sel::Union{Tuple, SpecialSelector})
     which = lowerselection(c, sel)
     cnames = colnames(c, which)
     if all(x->isa(x, Symbol), cnames)
-        tuplewrap = namedtuple(cnames...)
+        tuplewrap = namedtuple(cnames...)∘tuple
     else
         tuplewrap = tuple
     end
@@ -822,7 +832,7 @@ mutable struct ColDict{T}
     src::T
     names::Vector
     columns::Vector
-    copy::Union{Void, Bool}
+    copy::Union{Nothing, Bool}
 end
 
 """
@@ -834,7 +844,11 @@ To get the immutable iterator of the same type as `t`
 call `d[]`
 """
 function ColDict(t; copy=nothing)
-    ColDict(Int[], t, convert(Array{Any}, Base.copy(colnames(t))), Any[columns(t)...], copy)
+    cnames = colnames(t)
+    if cnames isa AbstractArray
+        cnames = Base.copy(cnames)
+    end
+    ColDict(Int[], t, convert(Array{Any}, collect(cnames)), Any[columns(t)...], copy)
 end
 
 function Base.getindex(d::ColDict{<:Columns})
@@ -1241,7 +1255,7 @@ function init_inputs(f, x, gettype, isvec) # normal functions
     f, x, gettype(f, x, isvec)
 end
 
-nicename(f) = Symbol(f)
+nicename(f) = Symbol(last(split(string(f), ".")))
 nicename(o::OnlineStat) = Symbol(typeof(o).name.name)
 
 function mapped_type(f, x, isvec)
@@ -1276,7 +1290,7 @@ function init_funcs(f::Tup, isvec)
         f
     end
 
-    namedtuple(ns...)(fs...), ss
+    NamedTuple{(ns...,)}((fs...,)), ss
 end
 
 function init_inputs(f::Tup, input, gettype, isvec)
@@ -1293,7 +1307,7 @@ function init_inputs(f::Tup, input, gettype, isvec)
     NT = namedtuple(ns...)
 
     # functions, input, output_eltype
-    NT(fs...), rows(NT(xs...)), NT{output_eltypes...}
+    NT((fs...,)), rows(NT((xs...,))), NT{Tuple{output_eltypes...}}
 end
 
 ### utils

src/flatten.jl

@@ -39,7 +39,7 @@ function mapslices(f, x::NDSparse, dims; name = nothing)
 
     if isa(y, NDSparse)
         # this means we need to concatenate outputs into a big NDSparse
-        ns = vcat(dimlabels(x)[iterdims], dimlabels(y))
+        ns = vcat(collect(dimlabels(x)[iterdims]), collect(dimlabels(y)))
         if !all(x->isa(x, Symbol), ns)
             ns = nothing
         else
@@ -128,8 +128,8 @@ function _mapslices_itable!(f, output, x, iter, iterdims, start)
     D = output.data
     initdims = length(iterdims)
 
-    I1 = Columns(I.columns[1:initdims]) # filled from existing table
-    I2 = Columns(I.columns[initdims+1:end]) # filled from output tables
+    I1 = Columns(getsubfields(I.columns, 1:initdims)) # filled from existing table
+    I2 = Columns(getsubfields(I.columns, initdims+1:nfields(I.columns))) # filled from output tables
 
     for i = start:length(iter)
         if i != 1 && roweq(iter, i-1, i) # We've already visited this slice
@@ -156,9 +156,17 @@ end
 function _flatten!(others, vecvec, out_others, out_vecvec)
     for i in 1:length(others)
         vec = vecvec[i]
-        for x in vec
-            push!(out_vecvec, x)
-            pushrow!(out_others, others, i)
+        try
+            for x in vec
+                push!(out_vecvec, x)
+                pushrow!(out_others, others, i)
+            end
+        catch err
+            # FIXME: this is slow!
+            if err isa MethodError && err.f === iterate
+                push!(out_vecvec, vec)
+                pushrow!(out_others, others, i)
+            end
         end
     end
 end
@@ -221,7 +229,7 @@ function flatten(t::NextTable, col=length(columns(t)); pkey=nothing)
     newcols = isa(cs, Tup) ? Any[cs...] : Any[cs]
     ns = colnames(out_vecvec)
     i = colindex(t, col)
-    cns = convert(Array{Any}, colnames(t))
+    cns = convert(Array{Any}, collect(colnames(t)))
     if length(ns) == 1 && !(ns[1] isa Symbol)
         ns = [colname(t, col)]
     end

src/indexing.jl

@@ -64,7 +64,7 @@ function _getindex(t::NDSparse, idxs)
     out = convert(Vector{Int32}, range_estimate(I, idxs))
     filter!(i->row_in(cs, i, idxs), out)
     keepdims = filter(i->eltype(columns(t.index)[i]) != typeof(idxs[i]), 1:length(idxs))
-    NDSparse(Columns(map(x->x[out], I.columns[keepdims])), t.data[out], presorted=true)
+    NDSparse(Columns(map(x->x[out], getsubfields(I.columns, keepdims))), t.data[out], presorted=true)
 end
 
 # iterators over indices - lazy getindex