Skip to content

Commit

Permalink
Fix bugs in append! and push! corner cases
Browse files Browse the repository at this point in the history
  • Loading branch information
@bkamins
bkamins committed Jun 1, 2020
1 parent da38804 commit 99b1143
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 12 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "DataFrames"
uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
version = "0.21.1"
version = "0.21.2"

[deps]
CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
Expand Down
27 changes: 19 additions & 8 deletions src/dataframe/dataframe.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1263,15 +1263,19 @@ function Base.append!(df1::DataFrame, df2::AbstractDataFrame; cols::Symbol=:sete
_columns(df1)[j] = newcol
end
else
# we can safely do it - if we got here this means that this is allowed
if Missing <: eltype(df1[!, j]) || !promote
append!(df1[!, j], df2[!, n])
else
if Missing <: eltype(df1[!, j])
resize!(df1[!, j], targetrows)
df1[nrows+1:targetrows, j] .= missing
elseif promote
newcol = similar(df1[!, j], Union{Missing, eltype(df1[!, j])},
targetrows)
copyto!(newcol, 1, df1[!, j], 1, nrows)
newcol[nrows+1:targetrows] .= missing
_columns(df1)[j] = newcol
else
throw(ArgumentError("promote=false and source data frame does " *
"not contain column :$n, while destination " *
"column does not allow for missing values"))
end
end
end
Expand Down Expand Up @@ -1350,9 +1354,18 @@ function Base.push!(df::DataFrame, row::Union{AbstractDict, NamedTuple};
end
S = typeof(val)
T = eltype(col)
if S <: T || !promote || promote_type(S, T) <: T
# if S <: T || promote_type(S, T) <: T this should never throw an exception
if S <: T || promote_type(S, T) <: T
push!(col, val)
elseif !promote
try
push!(col, val)
catch err
for col in _columns(df)
resize!(col, nrows)
end
@error "Error adding value to column :$colname."
rethrow(err)
end
else
newcol = similar(col, promote_type(S, T), targetrows)
copyto!(newcol, 1, col, 1, nrows)
Expand Down Expand Up @@ -1417,7 +1430,6 @@ function Base.push!(df::DataFrame, row::Union{AbstractDict, NamedTuple};
S = typeof(val)
T = eltype(col)
if S <: T || !promote || promote_type(S, T) <: T
# if S <: T || promote_type(S, T) <: T this should never throw an exception
push!(col, val)
else
newcol = similar(col, promote_type(S, T), targetrows)
Expand Down Expand Up @@ -1572,7 +1584,6 @@ function Base.push!(df::DataFrame, row::Any; promote::Bool=false)
S = typeof(val)
T = eltype(col)
if S <: T || !promote || promote_type(S, T) <: T
# if S <: T || promote_type(S, T) <: T this should never throw an exception
push!(col, val)
else
newcol = Tables.allocatecolumn(promote_type(S, T), targetrows)
Expand Down
14 changes: 11 additions & 3 deletions src/dataframerow/dataframerow.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,9 +403,18 @@ function Base.push!(df::DataFrame, dfr::DataFrameRow; cols::Symbol=:setequal,
end
S = typeof(val)
T = eltype(col)
if S <: T || !promote || promote_type(S, T) <: T
# if S <: T || promote_type(S, T) <: T this should never throw an exception
if S <: T || promote_type(S, T) <: T
push!(col, val)
elseif !promote
try
push!(col, val)
catch err
for col in _columns(df)
resize!(col, nrows)
end
@error "Error adding value to column :$colname."
rethrow(err)
end
else
newcol = Tables.allocatecolumn(promote_type(S, T), targetrows)
copyto!(newcol, 1, col, 1, nrows)
Expand Down Expand Up @@ -463,7 +472,6 @@ function Base.push!(df::DataFrame, dfr::DataFrameRow; cols::Symbol=:setequal,
S = typeof(val)
T = eltype(col)
if S <: T || !promote || promote_type(S, T) <: T
# if S <: T || promote_type(S, T) <: T this should never throw an exception
push!(col, val)
else
newcol = similar(col, promote_type(S, T), targetrows)
Expand Down
52 changes: 52 additions & 0 deletions test/dataframe.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,58 @@ end
end
end

@testset "new append! and push! tests" begin
for df in [DataFrame(a=Any[1]), DataFrame(a=1)]
@test append!(df, DataFrame(b=1), cols=:union) ≅
DataFrame(a=[1, missing], b=[missing, 1])
@test append!(df, DataFrame(b=1), cols=:union) ≅
DataFrame(a=[1, missing, missing], b=[missing, 1, 1])
df.x = 1:3
with_logger(SimpleLogger(IOBuffer())) do
@test_throws ArgumentError append!(df, DataFrame(b=1), cols=:union,
promote=false)
end
@test df ≅ DataFrame(a=[1, missing, missing], b=[missing, 1, 1], x=1:3)
allowmissing!(df, :x)
@test append!(df, DataFrame(b=1), cols=:union, promote=false) ≅
DataFrame(a=[1, missing, missing, missing], b=[missing, 1, 1, 1],
x=[1:3; missing])
end

for df in [DataFrame(a=Any[1]), DataFrame(a=1)]
@test push!(df, (b=1,), cols=:union) ≅
DataFrame(a=[1, missing], b=[missing, 1])
@test push!(df, (b=1,), cols=:union) ≅
DataFrame(a=[1, missing, missing], b=[missing, 1, 1])
df.x = 1:3
with_logger(SimpleLogger(IOBuffer())) do
@test_throws MethodError push!(df, (b=1,), cols=:union, promote=false)
end
@test df ≅ DataFrame(a=[1, missing, missing], b=[missing, 1, 1], x=1:3)
allowmissing!(df, :x)
@test push!(df, (b=1,), cols=:union, promote=false) ≅
DataFrame(a=[1, missing, missing, missing], b=[missing, 1, 1, 1],
x=[1:3; missing])
end

for df in [DataFrame(a=Any[1]), DataFrame(a=1)]
@test push!(df, DataFrame(b=1)[1, :], cols=:union) ≅
DataFrame(a=[1, missing], b=[missing, 1])
@test push!(df, DataFrame(b=1)[1, :], cols=:union) ≅
DataFrame(a=[1, missing, missing], b=[missing, 1, 1])
df.x = 1:3
with_logger(SimpleLogger(IOBuffer())) do
@test_throws MethodError push!(df, DataFrame(b=1)[1, :], cols=:union,
promote=false)
end
@test df ≅ DataFrame(a=[1, missing, missing], b=[missing, 1, 1], x=1:3)
allowmissing!(df, :x)
@test push!(df, DataFrame(b=1)[1, :], cols=:union, promote=false) ≅
DataFrame(a=[1, missing, missing, missing], b=[missing, 1, 1, 1],
x=[1:3; missing])
end
end

@testset "test categorical!" begin
df = DataFrame(A = Vector{Union{Int, Missing}}(1:3), B = Vector{Union{Int, Missing}}(4:6))
DRT = CategoricalArrays.DefaultRefType
Expand Down

2 comments on commit 99b1143

@bkamins
Copy link
Member Author

@bkamins bkamins commented on 99b1143 Jun 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/15695

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.21.2 -m "<description of version>" 99b114344c3aa40de9b81bf40c2335aa74bb685c
git push origin v0.21.2

Please sign in to comment.